Examples with UploadEntity com.cloud.storage.template.UploadEntity used on opensource projects

Example 1 with UploadEntity

use of com.cloud.storage.template.UploadEntity in project cosmic by MissionCriticalCloud.

the class NfsSecondaryStorageResource method createUploadEntity.

public UploadEntity createUploadEntity(String uuid, final String metadata, final long contentLength) {
    final TemplateOrVolumePostUploadCommand cmd = getTemplateOrVolumePostUploadCmd(metadata);
    UploadEntity uploadEntity = null;
    if (cmd == null) {
        final String errorMessage = "unable decode and deserialize metadata.";
        updateStateMapWithError(uuid, errorMessage);
        throw new InvalidParameterValueException(errorMessage);
    } else {
        uuid = cmd.getEntityUUID();
        if (isOneTimePostUrlUsed(cmd)) {
            uploadEntity = uploadEntityStateMap.get(uuid);
            final StringBuilder errorMessage = new StringBuilder("The one time post url is already used");
            if (uploadEntity != null) {
                errorMessage.append(" and the upload is in ").append(uploadEntity.getUploadState()).append(" state.");
            }
            throw new InvalidParameterValueException(errorMessage.toString());
        }
        final int maxSizeInGB = Integer.parseInt(cmd.getMaxUploadSize());
        final int contentLengthInGB = getSizeInGB(contentLength);
        if (contentLengthInGB > maxSizeInGB) {
            final String errorMessage = "Maximum file upload size exceeded. Content Length received: " + contentLengthInGB + "GB. Maximum allowed size: " + maxSizeInGB + "GB.";
            updateStateMapWithError(uuid, errorMessage);
            throw new InvalidParameterValueException(errorMessage);
        }
        checkSecondaryStorageResourceLimit(cmd, contentLengthInGB);
        try {
            final String absolutePath = cmd.getAbsolutePath();
            uploadEntity = new UploadEntity(uuid, cmd.getEntityId(), UploadEntity.Status.IN_PROGRESS, cmd.getName(), absolutePath);
            uploadEntity.setMetaDataPopulated(true);
            uploadEntity.setResourceType(UploadEntity.ResourceType.valueOf(cmd.getType()));
            uploadEntity.setFormat(Storage.ImageFormat.valueOf(cmd.getImageFormat()));
            // relative path with out ssvm mount info.
            uploadEntity.setTemplatePath(absolutePath);
            final String dataStoreUrl = cmd.getDataTo();
            final String installPathPrefix = getRootDir(dataStoreUrl) + File.separator + absolutePath;
            uploadEntity.setInstallPathPrefix(installPathPrefix);
            uploadEntity.setHvm(cmd.getRequiresHvm());
            uploadEntity.setChksum(cmd.getChecksum());
            uploadEntity.setMaxSizeInGB(maxSizeInGB);
            uploadEntity.setDescription(cmd.getDescription());
            uploadEntity.setContentLength(contentLength);
            // create a install dir
            if (!_storage.exists(installPathPrefix)) {
                _storage.mkdir(installPathPrefix);
            }
            uploadEntityStateMap.put(uuid, uploadEntity);
        } catch (final Exception e) {
            // upload entity will be null incase an exception occurs and the handler will not proceed.
            s_logger.error("exception occurred while creating upload entity ", e);
            updateStateMapWithError(uuid, e.getMessage());
        }
    }
    return uploadEntity;
}

Example 2 with UploadEntity

use of com.cloud.storage.template.UploadEntity in project cosmic by MissionCriticalCloud.

the class NfsSecondaryStorageResource method postUpload.

public String postUpload(final String uuid, final String filename) {
    final UploadEntity uploadEntity = uploadEntityStateMap.get(uuid);
    final int installTimeoutPerGig = 180 * 60 * 1000;
    final String resourcePath = uploadEntity.getInstallPathPrefix();
    // template download
    final String finalResourcePath = uploadEntity.getTmpltPath();
    final UploadEntity.ResourceType resourceType = uploadEntity.getResourceType();
    final String fileSavedTempLocation = uploadEntity.getInstallPathPrefix() + "/" + filename;
    final String uploadedFileExtension = FilenameUtils.getExtension(filename);
    String userSelectedFormat = uploadEntity.getFormat().toString();
    if (uploadedFileExtension.equals("zip") || uploadedFileExtension.equals("bz2") || uploadedFileExtension.equals("gz")) {
        userSelectedFormat += "." + uploadedFileExtension;
    }
    final String formatError = ImageStoreUtil.checkTemplateFormat(fileSavedTempLocation, userSelectedFormat);
    if (StringUtils.isNotBlank(formatError)) {
        final String errorString = "File type mismatch between uploaded file and selected format. Selected file format: " + userSelectedFormat + ". Received: " + formatError;
        s_logger.error(errorString);
        return errorString;
    }
    int imgSizeGigs = getSizeInGB(_storage.getSize(fileSavedTempLocation));
    final int maxSize = uploadEntity.getMaxSizeInGB();
    if (imgSizeGigs > maxSize) {
        final String errorMessage = "Maximum file upload size exceeded. Physical file size: " + imgSizeGigs + "GB. Maximum allowed size: " + maxSize + "GB.";
        s_logger.error(errorMessage);
        return errorMessage;
    }
    // add one just in case
    imgSizeGigs++;
    final long timeout = (long) imgSizeGigs * installTimeoutPerGig;
    final Script scr = new Script(getScriptLocation(resourceType), timeout, s_logger);
    scr.add("-s", Integer.toString(imgSizeGigs));
    scr.add("-S", Long.toString(UploadEntity.s_maxTemplateSize));
    if (uploadEntity.getDescription() != null && uploadEntity.getDescription().length() > 1) {
        scr.add("-d", uploadEntity.getDescription());
    }
    if (uploadEntity.isHvm()) {
        scr.add("-h");
    }
    final String checkSum = uploadEntity.getChksum();
    if (StringUtils.isNotBlank(checkSum)) {
        scr.add("-c", checkSum);
    }
    // add options common to ISO and template
    final String extension = uploadEntity.getFormat().getFileExtension();
    String templateName = "";
    if (extension.equals("iso")) {
        templateName = uploadEntity.getUuid().trim().replace(" ", "_");
    } else {
        try {
            templateName = UUID.nameUUIDFromBytes((uploadEntity.getFilename() + System.currentTimeMillis()).getBytes("UTF-8")).toString();
        } catch (final UnsupportedEncodingException e) {
            templateName = uploadEntity.getUuid().trim().replace(" ", "_");
        }
    }
    // run script to mv the temporary template file to the final template
    // file
    final String templateFilename = templateName + "." + extension;
    uploadEntity.setTemplatePath(finalResourcePath + "/" + templateFilename);
    scr.add("-n", templateFilename);
    scr.add("-t", resourcePath);
    // this is the temporary
    scr.add("-f", fileSavedTempLocation);
    // template file downloaded
    if (uploadEntity.getChksum() != null && uploadEntity.getChksum().length() > 1) {
        scr.add("-c", uploadEntity.getChksum());
    }
    // cleanup
    scr.add("-u");
    final String result;
    result = scr.execute();
    if (result != null) {
        return result;
    }
    // Set permissions for the downloaded template
    final File downloadedTemplate = new File(resourcePath + "/" + templateFilename);
    _storage.setWorldReadableAndWriteable(downloadedTemplate);
    // Set permissions for template/volume.properties
    String propertiesFile = resourcePath;
    if (resourceType == UploadEntity.ResourceType.TEMPLATE) {
        propertiesFile += "/template.properties";
    } else {
        propertiesFile += "/volume.properties";
    }
    final File templateProperties = new File(propertiesFile);
    _storage.setWorldReadableAndWriteable(templateProperties);
    final TemplateLocation loc = new TemplateLocation(_storage, resourcePath);
    try {
        loc.create(uploadEntity.getEntityId(), true, uploadEntity.getFilename());
    } catch (final IOException e) {
        s_logger.warn("Something is wrong with template location " + resourcePath, e);
        loc.purge();
        return "Unable to upload due to " + e.getMessage();
    }
    final Map<String, Processor> processors = _dlMgr.getProcessors();
    for (final Processor processor : processors.values()) {
        FormatInfo info = null;
        try {
            info = processor.process(resourcePath, null, templateName);
        } catch (final InternalErrorException e) {
            s_logger.error("Template process exception ", e);
            return e.toString();
        }
        if (info != null) {
            loc.addFormat(info);
            uploadEntity.setVirtualSize(info.virtualSize);
            uploadEntity.setPhysicalSize(info.size);
            break;
        }
    }
    if (!loc.save()) {
        s_logger.warn("Cleaning up because we're unable to save the formats");
        loc.purge();
    }
    uploadEntity.setStatus(UploadEntity.Status.COMPLETED);
    uploadEntityStateMap.put(uploadEntity.getUuid(), uploadEntity);
    return null;
}

Example 3 with UploadEntity

use of com.cloud.storage.template.UploadEntity in project cosmic by MissionCriticalCloud.

the class HttpUploadServerHandler method channelRead0.

@Override
public void channelRead0(final ChannelHandlerContext ctx, final HttpObject msg) throws Exception {
    if (msg instanceof HttpRequest) {
        final HttpRequest request = this.request = (HttpRequest) msg;
        responseContent.setLength(0);
        if (request.getMethod().equals(HttpMethod.POST)) {
            final URI uri = new URI(request.getUri());
            String signature = null;
            String expires = null;
            String metadata = null;
            String hostname = null;
            long contentLength = 0;
            for (final Entry<String, String> entry : request.headers()) {
                switch(entry.getKey()) {
                    case HEADER_SIGNATURE:
                        signature = entry.getValue();
                        break;
                    case HEADER_METADATA:
                        metadata = entry.getValue();
                        break;
                    case HEADER_EXPIRES:
                        expires = entry.getValue();
                        break;
                    case HEADER_HOST:
                        hostname = entry.getValue();
                        break;
                    case HttpHeaders.Names.CONTENT_LENGTH:
                        contentLength = Long.parseLong(entry.getValue());
                        break;
                }
            }
            logger.info("HEADER: signature=" + signature);
            logger.info("HEADER: metadata=" + metadata);
            logger.info("HEADER: expires=" + expires);
            logger.info("HEADER: hostname=" + hostname);
            logger.info("HEADER: Content-Length=" + contentLength);
            final QueryStringDecoder decoderQuery = new QueryStringDecoder(uri);
            final Map<String, List<String>> uriAttributes = decoderQuery.parameters();
            uuid = uriAttributes.get("uuid").get(0);
            logger.info("URI: uuid=" + uuid);
            UploadEntity uploadEntity = null;
            try {
                // Validate the request here
                storageResource.validatePostUploadRequest(signature, metadata, expires, hostname, contentLength, uuid);
                // create an upload entity. This will fail if entity already exists.
                uploadEntity = storageResource.createUploadEntity(uuid, metadata, contentLength);
            } catch (final InvalidParameterValueException ex) {
                logger.error("post request validation failed", ex);
                responseContent.append(ex.getMessage());
                writeResponse(ctx.channel(), HttpResponseStatus.BAD_REQUEST);
                requestProcessed = true;
                return;
            }
            if (uploadEntity == null) {
                logger.error("Unable to create upload entity. An exception occurred.");
                responseContent.append("Internal Server Error");
                writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
                requestProcessed = true;
                return;
            }
            // set the base directory to download the file
            DiskFileUpload.baseDirectory = uploadEntity.getInstallPathPrefix();
            logger.info("base directory: " + DiskFileUpload.baseDirectory);
            try {
                // initialize the decoder
                decoder = new HttpPostRequestDecoder(factory, request);
            } catch (ErrorDataDecoderException | IncompatibleDataDecoderException e) {
                logger.error("exception while initialising the decoder", e);
                responseContent.append(e.getMessage());
                writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
                requestProcessed = true;
                return;
            }
        } else {
            logger.warn("received a get request");
            responseContent.append("only post requests are allowed");
            writeResponse(ctx.channel(), HttpResponseStatus.BAD_REQUEST);
            requestProcessed = true;
            return;
        }
    }
    // check if the decoder was constructed before
    if (decoder != null) {
        if (msg instanceof HttpContent) {
            // New chunk is received
            final HttpContent chunk = (HttpContent) msg;
            try {
                decoder.offer(chunk);
            } catch (final ErrorDataDecoderException e) {
                logger.error("data decoding exception", e);
                responseContent.append(e.getMessage());
                writeResponse(ctx.channel(), HttpResponseStatus.INTERNAL_SERVER_ERROR);
                requestProcessed = true;
                return;
            }
            if (chunk instanceof LastHttpContent) {
                writeResponse(ctx.channel(), readFileUploadData());
                reset();
            }
        }
    }
}

Example 4 with UploadEntity

use of com.cloud.storage.template.UploadEntity in project cosmic by MissionCriticalCloud.

the class NfsSecondaryStorageResource method updateStateMapWithError.

public void updateStateMapWithError(final String uuid, final String errorMessage) {
    UploadEntity uploadEntity = null;
    if (uploadEntityStateMap.get(uuid) != null) {
        uploadEntity = uploadEntityStateMap.get(uuid);
    } else {
        uploadEntity = new UploadEntity();
    }
    uploadEntity.setStatus(UploadEntity.Status.ERROR);
    uploadEntity.setErrorMessage(errorMessage);
    uploadEntityStateMap.put(uuid, uploadEntity);
}

Example 5 with UploadEntity

use of com.cloud.storage.template.UploadEntity in project cosmic by MissionCriticalCloud.

the class NfsSecondaryStorageResource method execute.

private UploadStatusAnswer execute(final UploadStatusCommand cmd) {
    final String entityUuid = cmd.getEntityUuid();
    if (uploadEntityStateMap.containsKey(entityUuid)) {
        final UploadEntity uploadEntity = uploadEntityStateMap.get(entityUuid);
        if (uploadEntity.getUploadState() == UploadEntity.Status.ERROR) {
            uploadEntityStateMap.remove(entityUuid);
            return new UploadStatusAnswer(cmd, UploadStatus.ERROR, uploadEntity.getErrorMessage());
        } else if (uploadEntity.getUploadState() == UploadEntity.Status.COMPLETED) {
            final UploadStatusAnswer answer = new UploadStatusAnswer(cmd, UploadStatus.COMPLETED);
            answer.setVirtualSize(uploadEntity.getVirtualSize());
            answer.setInstallPath(uploadEntity.getTmpltPath());
            answer.setPhysicalSize(uploadEntity.getPhysicalSize());
            answer.setDownloadPercent(100);
            uploadEntityStateMap.remove(entityUuid);
            return answer;
        } else if (uploadEntity.getUploadState() == UploadEntity.Status.IN_PROGRESS) {
            final UploadStatusAnswer answer = new UploadStatusAnswer(cmd, UploadStatus.IN_PROGRESS);
            final long downloadedSize = FileUtils.sizeOfDirectory(new File(uploadEntity.getInstallPathPrefix()));
            final int downloadPercent = (int) (100 * downloadedSize / uploadEntity.getContentLength());
            answer.setDownloadPercent(Math.min(downloadPercent, 100));
            return answer;
        }
    }
    return new UploadStatusAnswer(cmd, UploadStatus.UNKNOWN);
}