Search in sources :

Example 1 with Part

use of com.pspace.ifs.ksan.gw.object.multipart.Part in project ksan by infinistor.

the class AbortMultipartUpload method process.

@Override
public void process() throws GWException {
    logger.info(GWConstants.LOG_ABORT_MULTIPART_UPLOAD_START);
    String bucket = s3Parameter.getBucketName();
    initBucketInfo(bucket);
    String object = s3Parameter.getObjectName();
    S3Bucket s3Bucket = new S3Bucket();
    s3Bucket.setCors(getBucketInfo().getCors());
    s3Bucket.setAccess(getBucketInfo().getAccess());
    s3Parameter.setBucket(s3Bucket);
    GWUtils.checkCors(s3Parameter);
    if (s3Parameter.isPublicAccess() && GWUtils.isIgnorePublicAcls(s3Parameter)) {
        throw new GWException(GWErrorCode.ACCESS_DENIED, s3Parameter);
    }
    DataAbortMultipartUpload dataAbortMultipartUpload = new DataAbortMultipartUpload(s3Parameter);
    dataAbortMultipartUpload.extract();
    String uploadId = dataAbortMultipartUpload.getUploadId();
    s3Parameter.setUploadId(uploadId);
    ObjMultipart objMultipart = null;
    SortedMap<Integer, Part> listPart = null;
    boolean isUploadId = true;
    try {
        objMultipart = new ObjMultipart(bucket);
        if (!objMultipart.isUploadId(uploadId)) {
            isUploadId = false;
            throw new GWException(GWErrorCode.NO_SUCH_UPLOAD, s3Parameter);
        }
        listPart = objMultipart.getParts(uploadId);
    } catch (Exception e) {
        if (!isUploadId) {
            throw new GWException(GWErrorCode.NO_SUCH_UPLOAD, s3Parameter);
        }
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    }
    // get Paths
    Metadata objMeta = createLocal(bucket, object);
    S3ObjectOperation objectOperation = new S3ObjectOperation(objMeta, null, s3Parameter, null, null);
    objectOperation.abortMultipart(listPart);
    objMultipart.abortMultipartUpload(uploadId);
    s3Parameter.getResponse().setStatus(HttpServletResponse.SC_NO_CONTENT);
}
Also used : ObjMultipart(com.pspace.ifs.ksan.objmanager.ObjMultipart) S3ObjectOperation(com.pspace.ifs.ksan.gw.object.S3ObjectOperation) S3Bucket(com.pspace.ifs.ksan.gw.identity.S3Bucket) DataAbortMultipartUpload(com.pspace.ifs.ksan.gw.data.DataAbortMultipartUpload) Part(com.pspace.ifs.ksan.gw.object.multipart.Part) Metadata(com.pspace.ifs.ksan.objmanager.Metadata) GWException(com.pspace.ifs.ksan.gw.exception.GWException) GWException(com.pspace.ifs.ksan.gw.exception.GWException)

Example 2 with Part

use of com.pspace.ifs.ksan.gw.object.multipart.Part in project ksan by infinistor.

the class CompleteMultipartUpload method process.

@Override
public void process() throws GWException {
    logger.info(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_START);
    String bucket = s3Parameter.getBucketName();
    initBucketInfo(bucket);
    String object = s3Parameter.getObjectName();
    S3Bucket s3Bucket = new S3Bucket();
    s3Bucket.setCors(getBucketInfo().getCors());
    s3Bucket.setAccess(getBucketInfo().getAccess());
    s3Parameter.setBucket(s3Bucket);
    GWUtils.checkCors(s3Parameter);
    if (s3Parameter.isPublicAccess() && GWUtils.isIgnorePublicAcls(s3Parameter)) {
        throw new GWException(GWErrorCode.ACCESS_DENIED, s3Parameter);
    }
    DataCompleteMultipartUpload dataCompleteMultipartUpload = new DataCompleteMultipartUpload(s3Parameter);
    dataCompleteMultipartUpload.extract();
    String uploadId = dataCompleteMultipartUpload.getUploadId();
    s3Parameter.setUploadId(uploadId);
    String multipartXml = dataCompleteMultipartUpload.getMultipartXml();
    XmlMapper xmlMapper = new XmlMapper();
    CompleteMultipartUploadRequest completeMultipartUpload = new CompleteMultipartUploadRequest();
    try {
        completeMultipartUpload = xmlMapper.readValue(multipartXml, CompleteMultipartUploadRequest.class);
    } catch (JsonMappingException e) {
        PrintStack.logging(logger, e);
        new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    } catch (JsonProcessingException e) {
        PrintStack.logging(logger, e);
        new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    }
    if (completeMultipartUpload.parts == null || completeMultipartUpload.parts.size() == 0) {
        logger.error(GWErrorCode.MALFORMED_X_M_L.getMessage() + GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_PART_NO_EXIST);
        throw new GWException(GWErrorCode.MALFORMED_X_M_L, s3Parameter);
    }
    SortedMap<Integer, Part> xmlListPart = new TreeMap<Integer, Part>();
    for (CompleteMultipartUploadRequest.Part sPart : completeMultipartUpload.parts) {
        Part op = new Part();
        op.setPartETag(sPart.eTag);
        op.setPartNumber(sPart.partNumber);
        xmlListPart.put(sPart.partNumber, op);
    }
    SortedMap<Integer, Part> listPart = null;
    ObjMultipart objMultipart = null;
    try {
        objMultipart = new ObjMultipart(bucket);
        listPart = objMultipart.getParts(uploadId);
    } catch (UnknownHostException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    } catch (Exception e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    }
    logger.info(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_XML_PARTS_SIZE, completeMultipartUpload.parts.size());
    logger.info(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_PARTS_SIZE, listPart.size());
    if (completeMultipartUpload.parts.size() > listPart.size()) {
        logger.warn(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_LESS_THAN);
        throw new GWException(GWErrorCode.INVALID_PART, s3Parameter);
    } else if (completeMultipartUpload.parts.size() < listPart.size()) {
        logger.warn(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_GREATER_THAN);
        throw new GWException(GWErrorCode.INVALID_PART, s3Parameter);
    }
    for (Iterator<Map.Entry<Integer, Part>> it = xmlListPart.entrySet().iterator(); it.hasNext(); ) {
        Map.Entry<Integer, Part> entry = it.next();
        String eTag = entry.getValue().getPartETag().replace(GWConstants.DOUBLE_QUOTE, "");
        if (listPart.containsKey(entry.getKey())) {
            if (eTag.compareTo(listPart.get(entry.getKey()).getPartETag()) == 0) {
                if (listPart.get(entry.getKey()).getPartSize() < GWConstants.PARTS_MIN_SIZE && entry.getKey() < listPart.size()) {
                    logger.error(GWErrorCode.ENTITY_TOO_SMALL.getMessage());
                    throw new GWException(GWErrorCode.ENTITY_TOO_SMALL, s3Parameter);
                }
            } else {
                throw new GWException(GWErrorCode.INVALID_PART, s3Parameter);
            }
        } else {
            throw new GWException(GWErrorCode.INVALID_PART, s3Parameter);
        }
    }
    // get Acl, Meta data
    Multipart multipart = null;
    try {
        objMultipart = new ObjMultipart(bucket);
        multipart = objMultipart.getMultipart(uploadId);
        if (multipart == null) {
            logger.error(GWConstants.LOG_UPLOAD_NOT_FOUND, uploadId);
            throw new GWException(GWErrorCode.NO_SUCH_UPLOAD, s3Parameter);
        }
    } catch (UnknownHostException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    } catch (Exception e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    }
    String acl = multipart.getAcl();
    String meta = multipart.getMeta();
    // check bucket versioning, and set versionId
    String versioningStatus = getBucketVersioning(bucket);
    String versionId = null;
    Metadata objMeta = null;
    try {
        // check exist object
        objMeta = open(bucket, object);
        if (GWConstants.VERSIONING_ENABLED.equalsIgnoreCase(versioningStatus)) {
            versionId = String.valueOf(System.nanoTime());
        } else {
            versionId = GWConstants.VERSIONING_DISABLE_TAIL;
        }
    } catch (GWException e) {
        logger.info(e.getMessage());
        objMeta = createLocal(bucket, object);
        if (GWConstants.VERSIONING_ENABLED.equalsIgnoreCase(versioningStatus)) {
            versionId = String.valueOf(System.nanoTime());
        } else {
            versionId = GWConstants.VERSIONING_DISABLE_TAIL;
        }
    }
    if (GWConstants.VERSIONING_ENABLED.equalsIgnoreCase(versioningStatus)) {
        logger.info(GWConstants.LOG_COMPLETE_MULTIPART_VERSION_ID, versionId);
        s3Parameter.getResponse().addHeader(GWConstants.X_AMZ_VERSION_ID, versionId);
    }
    s3Parameter.getResponse().setCharacterEncoding(GWConstants.CHARSET_UTF_8);
    XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
    try (Writer writer = s3Parameter.getResponse().getWriter()) {
        s3Parameter.getResponse().setContentType(GWConstants.XML_CONTENT_TYPE);
        final AtomicReference<S3Object> s3Object = new AtomicReference<>();
        final AtomicReference<Exception> S3Excp = new AtomicReference<>();
        S3ObjectOperation objectOperation = new S3ObjectOperation(objMeta, null, s3Parameter, versionId, null);
        ObjectMapper jsonMapper = new ObjectMapper();
        S3Metadata s3Metadata = jsonMapper.readValue(meta, S3Metadata.class);
        SortedMap<Integer, Part> constListPart = listPart;
        Thread thread = new Thread() {

            @Override
            public void run() {
                try {
                    s3Object.set(objectOperation.completeMultipart(constListPart));
                } catch (Exception e) {
                    S3Excp.set(e);
                }
            }
        };
        thread.start();
        XMLStreamWriter xmlStreamWriter = xmlOutputFactory.createXMLStreamWriter(writer);
        xmlStreamWriter.writeStartDocument();
        xmlStreamWriter.writeStartElement(GWConstants.COMPLETE_MULTIPART_UPLOAD_RESULT);
        xmlStreamWriter.writeDefaultNamespace(GWConstants.AWS_XMLNS);
        while (thread.isAlive()) {
            try {
                thread.join(500);
            } catch (InterruptedException ie) {
                PrintStack.logging(logger, ie);
                throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
            }
            xmlStreamWriter.writeCharacters(GWConstants.NEWLINE);
        }
        if (S3Excp.get() != null) {
            PrintStack.logging(logger, S3Excp.get());
            logger.error(S3Excp.get().getMessage());
            throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
        }
        writeSimpleElement(xmlStreamWriter, GWConstants.LOCATION, GWConstants.HTTP + bucket + GWConstants.S3_AMAZON_AWS_COM + object);
        writeSimpleElement(xmlStreamWriter, GWConstants.BUCKET, bucket);
        writeSimpleElement(xmlStreamWriter, GWConstants.KEY, object);
        // make ETag
        StringBuilder sb = new StringBuilder();
        for (Iterator<Map.Entry<Integer, Part>> it = listPart.entrySet().iterator(); it.hasNext(); ) {
            Map.Entry<Integer, Part> entry = it.next();
            sb.append(entry.getValue().getPartETag());
        }
        String hex = sb.toString();
        byte[] raw = BaseEncoding.base16().decode(hex.toUpperCase());
        Hasher hasher = Hashing.md5().newHasher();
        hasher.putBytes(raw);
        String digest = hasher.hash().toString() + GWConstants.DASH + listPart.size();
        s3Object.get().setEtag(digest);
        logger.debug(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_MD5, s3Object.get().getEtag());
        writeSimpleElement(xmlStreamWriter, GWConstants.ETAG, GWConstants.DOUBLE_QUOTE + s3Object.get().getEtag() + GWConstants.DOUBLE_QUOTE);
        xmlStreamWriter.writeEndElement();
        xmlStreamWriter.flush();
        s3Metadata.setSize(s3Object.get().getFileSize());
        s3Metadata.setContentLength(s3Object.get().getFileSize());
        s3Metadata.setETag(s3Object.get().getEtag());
        s3Metadata.setLastModified(s3Object.get().getLastModified());
        s3Metadata.setTier(GWConstants.AWS_TIER_STANTARD);
        s3Metadata.setDeleteMarker(s3Object.get().getDeleteMarker());
        s3Metadata.setVersionId(s3Object.get().getVersionId());
        String jsonmeta = "";
        try {
            jsonmeta = jsonMapper.writeValueAsString(s3Metadata);
        } catch (JsonProcessingException e) {
            PrintStack.logging(logger, e);
            throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
        }
        int result = 0;
        try {
            remove(bucket, object);
            objMeta.set(s3Object.get().getEtag(), "", jsonmeta, acl, s3Object.get().getFileSize());
            objMeta.setVersionId(versionId, GWConstants.OBJECT_TYPE_FILE, true);
            result = insertObject(bucket, object, objMeta);
        } catch (GWException e) {
            PrintStack.logging(logger, e);
            throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
        }
        if (result != 0) {
            logger.error(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_FAILED, bucket, object);
        }
        logger.debug(GWConstants.LOG_COMPLETE_MULTIPART_UPLOAD_INFO, bucket, object, s3Object.get().getFileSize(), s3Object.get().getEtag(), acl, versionId);
        objMultipart.abortMultipartUpload(uploadId);
    } catch (IOException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    } catch (XMLStreamException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    }
}
Also used : ObjMultipart(com.pspace.ifs.ksan.objmanager.ObjMultipart) Multipart(com.pspace.ifs.ksan.gw.object.multipart.Multipart) ObjMultipart(com.pspace.ifs.ksan.objmanager.ObjMultipart) XMLOutputFactory(javax.xml.stream.XMLOutputFactory) S3Metadata(com.pspace.ifs.ksan.gw.identity.S3Metadata) Metadata(com.pspace.ifs.ksan.objmanager.Metadata) S3ObjectOperation(com.pspace.ifs.ksan.gw.object.S3ObjectOperation) S3Bucket(com.pspace.ifs.ksan.gw.identity.S3Bucket) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) GWException(com.pspace.ifs.ksan.gw.exception.GWException) S3Object(com.pspace.ifs.ksan.gw.object.S3Object) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) DataCompleteMultipartUpload(com.pspace.ifs.ksan.gw.data.DataCompleteMultipartUpload) UnknownHostException(java.net.UnknownHostException) AtomicReference(java.util.concurrent.atomic.AtomicReference) IOException(java.io.IOException) TreeMap(java.util.TreeMap) GWException(com.pspace.ifs.ksan.gw.exception.GWException) XMLStreamException(javax.xml.stream.XMLStreamException) ResourceNotFoundException(com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException) IOException(java.io.IOException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) UnknownHostException(java.net.UnknownHostException) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) XmlMapper(com.fasterxml.jackson.dataformat.xml.XmlMapper) Hasher(com.google.common.hash.Hasher) XMLStreamException(javax.xml.stream.XMLStreamException) Part(com.pspace.ifs.ksan.gw.object.multipart.Part) S3Metadata(com.pspace.ifs.ksan.gw.identity.S3Metadata) Map(java.util.Map) TreeMap(java.util.TreeMap) SortedMap(java.util.SortedMap) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) Writer(java.io.Writer) CompleteMultipartUploadRequest(com.pspace.ifs.ksan.gw.format.CompleteMultipartUploadRequest)

Example 3 with Part

use of com.pspace.ifs.ksan.gw.object.multipart.Part in project ksan by infinistor.

the class MongoDataRepository method getParts.

@Override
public SortedMap<Integer, Part> getParts(String uploadId) throws SQLException {
    SortedMap<Integer, Part> listPart = new TreeMap<>();
    MongoCollection<Document> multip;
    Multipart mpart = null;
    multip = getMultiPartUploadCollection();
    if (multip == null)
        return null;
    FindIterable fit = multip.find(Filters.eq(UPLOADID, uploadId));
    Iterator it = fit.iterator();
    while ((it.hasNext())) {
        Document doc = (Document) it.next();
        Part part = new Part();
        part.setLastModified((Date) doc.getDate(LASTMODIFIED));
        part.setPartETag(doc.getString(ETAG));
        part.setPartSize(doc.getLong(SIZE));
        part.setPartNumber(doc.getInteger(PARTNO));
        part.setDiskID(PDISKID);
        listPart.put(part.getPartNumber(), part);
    }
    return listPart;
}
Also used : Multipart(com.pspace.ifs.ksan.gw.object.multipart.Multipart) Part(com.pspace.ifs.ksan.gw.object.multipart.Part) Iterator(java.util.Iterator) FindIterable(com.mongodb.client.FindIterable) TreeMap(java.util.TreeMap) Document(org.bson.Document)

Example 4 with Part

use of com.pspace.ifs.ksan.gw.object.multipart.Part in project ksan by infinistor.

the class S3ObjectOperation method completeMultipart.

public S3Object completeMultipart(SortedMap<Integer, Part> listPart) throws Exception {
    S3Object s3Object = new S3Object();
    byte[] buffer = new byte[GWConstants.MAXBUFSIZE];
    MessageDigest md5er = MessageDigest.getInstance(GWConstants.MD5);
    long totalLength = 0L;
    long existFileSize = 0L;
    long putSize = 0L;
    long calSize = 0L;
    File tmpFile = new File(makeTempPath(GWDiskConfig.getInstance().getLocalPath(), objMeta.getObjId(), versionId));
    try (FileOutputStream tmpOut = new FileOutputStream(tmpFile)) {
        com.google.common.io.Files.createParentDirs(tmpFile);
        // for each part object
        for (Iterator<Map.Entry<Integer, Part>> it = listPart.entrySet().iterator(); it.hasNext(); ) {
            Map.Entry<Integer, Part> entry = it.next();
            if (GWDiskConfig.getInstance().getLocalDiskID().equals(entry.getValue().getDiskID())) {
                // part is in local disk
                File partFile = new File(makeTempPath(GWDiskConfig.getInstance().getLocalPath(), objMeta.getObjId(), String.valueOf(entry.getValue().getPartNumber())));
                try (FileInputStream fis = new FileInputStream(partFile)) {
                    int readLength = 0;
                    while ((readLength = fis.read(buffer, 0, GWConstants.MAXBUFSIZE)) != -1) {
                        totalLength += readLength;
                        tmpOut.write(buffer, 0, readLength);
                        md5er.update(buffer, 0, readLength);
                    }
                    tmpOut.flush();
                    if (!partFile.delete()) {
                        logger.error(GWConstants.LOG_S3OBJECT_OPERATION_FAILED_FILE_DELETE, partFile.getName());
                    }
                }
            } else {
                OSDClient client = OSDClientManager.getInstance().getOSDClient(entry.getValue().getDiskID());
                client.getPartInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), String.valueOf(entry.getValue().getPartNumber()), entry.getValue().getPartSize(), tmpOut, md5er);
                totalLength += client.getPart();
                OSDClientManager.getInstance().returnOSDClient(client);
            }
        }
        byte[] digest = md5er.digest();
        String eTag = base16().lowerCase().encode(digest);
        s3Object.setEtag(eTag);
        s3Object.setLastModified(new Date());
        s3Object.setFileSize(totalLength);
        s3Object.setDeleteMarker(GWConstants.OBJECT_TYPE_FILE);
    }
    OSDClient clientPrimary = null;
    OSDClient clientReplica = null;
    int readLength = 0;
    existFileSize = objMeta.getSize();
    putSize = totalLength;
    if (objMeta.getReplicaCount() > 1) {
        existFileSize *= objMeta.getReplicaCount();
        putSize *= objMeta.getReplicaCount();
        if (!GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
            clientPrimary = OSDClientManager.getInstance().getOSDClient(objMeta.getPrimaryDisk().getOsdIp());
            clientPrimary.putInit(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId, totalLength, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId(), GWConfig.getPerformanceMode());
        }
        if (!GWUtils.getLocalIP().equals(objMeta.getReplicaDisk().getOsdIp())) {
            clientReplica = OSDClientManager.getInstance().getOSDClient(objMeta.getReplicaDisk().getOsdIp());
            clientReplica.putInit(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId, totalLength, GWConstants.FILE_ATTRIBUTE_REPLICATION_REPLICA, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL, GWConfig.getPerformanceMode());
        }
        if (clientPrimary != null || clientReplica != null) {
            try (FileInputStream fis = new FileInputStream(tmpFile)) {
                readLength = 0;
                while ((readLength = fis.read(buffer, 0, GWConstants.MAXBUFSIZE)) != -1) {
                    if (clientPrimary != null) {
                        clientPrimary.put(buffer, 0, readLength);
                    }
                    if (clientReplica != null) {
                        clientReplica.put(buffer, 0, readLength);
                    }
                }
            }
        }
        if (GWUtils.getLocalIP().equals(objMeta.getPrimaryDisk().getOsdIp())) {
            File file = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
            File trashFile = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
            com.google.common.io.Files.createParentDirs(file);
            if (file.exists()) {
                retryRenameTo(file, trashFile);
            }
            setAttributeFileReplication(tmpFile, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, objMeta.getReplicaDisk().getId());
            retryRenameTo(tmpFile, file);
        }
        if (GWUtils.getLocalIP().equals(objMeta.getReplicaDisk().getOsdIp())) {
            File file = new File(makeObjPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
            File trashFile = new File(makeTrashPath(objMeta.getReplicaDisk().getPath(), objMeta.getObjId(), versionId));
            com.google.common.io.Files.createParentDirs(file);
            if (file.exists()) {
                retryRenameTo(file, trashFile);
            }
            setAttributeFileReplication(tmpFile, GWConstants.FILE_ATTRIBUTE_REPLICATION_REPLICA, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL);
            retryRenameTo(tmpFile, file);
        }
    } else {
        File file = new File(makeObjPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
        File trashFile = new File(makeTrashPath(objMeta.getPrimaryDisk().getPath(), objMeta.getObjId(), versionId));
        com.google.common.io.Files.createParentDirs(file);
        if (file.exists()) {
            retryRenameTo(file, trashFile);
        }
        setAttributeFileReplication(tmpFile, GWConstants.FILE_ATTRUBUTE_REPLICATION_PRIMARY, GWConstants.FILE_ATTRIBUTE_REPLICA_DISK_ID_NULL);
        retryRenameTo(tmpFile, file);
    }
    calSize = putSize - existFileSize;
    updateBucketUsed(objMeta.getBucket(), calSize);
    return s3Object;
}
Also used : FileInputStream(java.io.FileInputStream) Date(java.util.Date) OSDClient(com.pspace.ifs.ksan.gw.object.osdclient.OSDClient) Part(com.pspace.ifs.ksan.gw.object.multipart.Part) FileOutputStream(java.io.FileOutputStream) MessageDigest(java.security.MessageDigest) File(java.io.File) Map(java.util.Map) SortedMap(java.util.SortedMap)

Example 5 with Part

use of com.pspace.ifs.ksan.gw.object.multipart.Part in project ksan by infinistor.

the class ListParts method process.

@Override
public void process() throws GWException {
    logger.info(GWConstants.LOG_LIST_PARTS_START);
    String bucket = s3Parameter.getBucketName();
    if (!isExistBucket(bucket)) {
        logger.error(GWConstants.LOG_BUCKET_IS_NOT_EXIST, bucket);
        throw new GWException(GWErrorCode.NO_SUCH_BUCKET, s3Parameter);
    }
    initBucketInfo(bucket);
    S3Bucket s3Bucket = new S3Bucket();
    s3Bucket.setCors(getBucketInfo().getCors());
    s3Bucket.setAccess(getBucketInfo().getAccess());
    s3Parameter.setBucket(s3Bucket);
    GWUtils.checkCors(s3Parameter);
    if (s3Parameter.isPublicAccess() && GWUtils.isIgnorePublicAcls(s3Parameter)) {
        throw new GWException(GWErrorCode.ACCESS_DENIED, s3Parameter);
    }
    checkGrantBucket(s3Parameter.isPublicAccess(), String.valueOf(s3Parameter.getUser().getUserId()), GWConstants.GRANT_READ);
    DataListParts dataListParts = new DataListParts(s3Parameter);
    dataListParts.extract();
    String maxParts = dataListParts.getMaxParts();
    String partNumberMarker = dataListParts.getPartNumberMarker();
    String uploadId = dataListParts.getUploadId();
    int maxPartsValue = GWConstants.MAX_PART_VALUE;
    if (!Strings.isNullOrEmpty(maxParts)) {
        if (Integer.valueOf(maxParts) < 0) {
            throw new GWException(GWErrorCode.INVALID_ARGUMENT, s3Parameter);
        }
        maxPartsValue = Integer.valueOf(maxParts);
    }
    ResultParts resultPart = null;
    ObjMultipart objMultipart = null;
    Multipart multipart = null;
    try {
        objMultipart = new ObjMultipart(bucket);
        multipart = objMultipart.getMultipart(uploadId);
        if (multipart == null) {
            logger.error(GWConstants.LOG_UPLOAD_NOT_FOUND, uploadId);
            throw new GWException(GWErrorCode.NO_SUCH_UPLOAD, s3Parameter);
        }
        // check acl use multipart acl
        resultPart = objMultipart.getParts(uploadId, partNumberMarker, maxPartsValue);
    } catch (UnknownHostException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    } catch (Exception e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    }
    String meta = multipart.getMeta();
    ObjectMapper jsonMapper = new ObjectMapper();
    S3Metadata s3Metadata;
    try {
        s3Metadata = jsonMapper.readValue(meta, S3Metadata.class);
    } catch (JsonMappingException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    } catch (JsonProcessingException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.INTERNAL_SERVER_ERROR, s3Parameter);
    }
    XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newInstance();
    s3Parameter.getResponse().setCharacterEncoding(GWConstants.CHARSET_UTF_8);
    try (Writer writer = s3Parameter.getResponse().getWriter()) {
        s3Parameter.getResponse().setContentType(GWConstants.XML_CONTENT_TYPE);
        XMLStreamWriter xmlStreamWriter = xmlOutputFactory.createXMLStreamWriter(writer);
        xmlStreamWriter.writeStartDocument();
        xmlStreamWriter.writeStartElement(GWConstants.LIST_PARTS_RESULT);
        xmlStreamWriter.writeDefaultNamespace(GWConstants.AWS_XMLNS);
        writeSimpleElement(xmlStreamWriter, GWConstants.BUCKET, bucket);
        writeSimpleElement(xmlStreamWriter, GWConstants.KEY, s3Parameter.getObjectName());
        writeSimpleElement(xmlStreamWriter, GWConstants.XML_UPLOADID, uploadId);
        writeInitiatorStanza(xmlStreamWriter);
        writeOwnerInfini(xmlStreamWriter, s3Metadata.getOwnerId(), s3Metadata.getOwnerName());
        writeSimpleElement(xmlStreamWriter, GWConstants.STORAGE_CLASS, GWConstants.AWS_TIER_STANTARD);
        if (resultPart.isTruncated()) {
            writeSimpleElement(xmlStreamWriter, GWConstants.XML_IS_TRUNCATED, GWConstants.XML_TRUE);
            writeSimpleElement(xmlStreamWriter, GWConstants.XML_NEXT_PART_NUMBER, String.valueOf(resultPart.getPartNumberMarker()));
        } else {
            writeSimpleElement(xmlStreamWriter, GWConstants.XML_IS_TRUNCATED, GWConstants.XML_FALSE);
        }
        for (Iterator<Map.Entry<Integer, Part>> it = resultPart.getListPart().entrySet().iterator(); it.hasNext(); ) {
            Map.Entry<Integer, Part> entry = it.next();
            xmlStreamWriter.writeStartElement(GWConstants.PART);
            writeSimpleElement(xmlStreamWriter, GWConstants.PARTNUMBER, String.valueOf(entry.getValue().getPartNumber()));
            writeSimpleElement(xmlStreamWriter, GWConstants.LAST_MODIFIED, formatDate(entry.getValue().getLastModified()));
            writeSimpleElement(xmlStreamWriter, GWConstants.ETAG, entry.getValue().getPartETag());
            writeSimpleElement(xmlStreamWriter, GWConstants.XML_SIZE, String.valueOf(entry.getValue().getPartSize()));
            xmlStreamWriter.writeEndElement();
            logger.debug(GWConstants.LOG_LIST_PARTS_PART_NUMBER, entry.getValue().getPartNumber());
            logger.debug(GWConstants.LOG_LIST_PARTS_LAST_MODIFIED, formatDate(entry.getValue().getLastModified()));
            logger.debug(GWConstants.LOG_LIST_PARTS_ETAG, entry.getValue().getPartETag());
            logger.debug(GWConstants.LOG_LIST_PARTS_SIZE, String.valueOf(entry.getValue().getPartSize()));
        }
        xmlStreamWriter.writeEndElement();
        xmlStreamWriter.flush();
    } catch (IOException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    } catch (XMLStreamException e) {
        PrintStack.logging(logger, e);
        throw new GWException(GWErrorCode.SERVER_ERROR, s3Parameter);
    }
}
Also used : ObjMultipart(com.pspace.ifs.ksan.objmanager.ObjMultipart) Multipart(com.pspace.ifs.ksan.gw.object.multipart.Multipart) ObjMultipart(com.pspace.ifs.ksan.objmanager.ObjMultipart) XMLOutputFactory(javax.xml.stream.XMLOutputFactory) DataListParts(com.pspace.ifs.ksan.gw.data.DataListParts) S3Bucket(com.pspace.ifs.ksan.gw.identity.S3Bucket) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) GWException(com.pspace.ifs.ksan.gw.exception.GWException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) UnknownHostException(java.net.UnknownHostException) ResultParts(com.pspace.ifs.ksan.gw.object.multipart.ResultParts) IOException(java.io.IOException) GWException(com.pspace.ifs.ksan.gw.exception.GWException) XMLStreamException(javax.xml.stream.XMLStreamException) IOException(java.io.IOException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) UnknownHostException(java.net.UnknownHostException) JsonMappingException(com.fasterxml.jackson.databind.JsonMappingException) XMLStreamException(javax.xml.stream.XMLStreamException) S3Metadata(com.pspace.ifs.ksan.gw.identity.S3Metadata) Part(com.pspace.ifs.ksan.gw.object.multipart.Part) Map(java.util.Map) XMLStreamWriter(javax.xml.stream.XMLStreamWriter) Writer(java.io.Writer)

Aggregations

Part (com.pspace.ifs.ksan.gw.object.multipart.Part)9 GWException (com.pspace.ifs.ksan.gw.exception.GWException)4 Map (java.util.Map)4 S3Bucket (com.pspace.ifs.ksan.gw.identity.S3Bucket)3 Multipart (com.pspace.ifs.ksan.gw.object.multipart.Multipart)3 ResultParts (com.pspace.ifs.ksan.gw.object.multipart.ResultParts)3 ObjMultipart (com.pspace.ifs.ksan.objmanager.ObjMultipart)3 IOException (java.io.IOException)3 SortedMap (java.util.SortedMap)3 TreeMap (java.util.TreeMap)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 JsonMappingException (com.fasterxml.jackson.databind.JsonMappingException)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)2 FindIterable (com.mongodb.client.FindIterable)2 S3Metadata (com.pspace.ifs.ksan.gw.identity.S3Metadata)2 S3ObjectOperation (com.pspace.ifs.ksan.gw.object.S3ObjectOperation)2 OSDClient (com.pspace.ifs.ksan.gw.object.osdclient.OSDClient)2 Metadata (com.pspace.ifs.ksan.objmanager.Metadata)2 ResourceNotFoundException (com.pspace.ifs.ksan.objmanager.ObjManagerException.ResourceNotFoundException)2 File (java.io.File)2