Examples with CRC64 com.qcloud.cos.utils.CRC64 used on opensource projects

Example 1 with CRC64

use of com.qcloud.cos.utils.CRC64 in project cos-java-sdk-v5 by tencentyun.

the class ResumableDownloadSubmitter method submit.

public void submit() throws Exception {
    long contentLength = Long.parseLong(downloadRecord.getContentLength());
    download.setState(TransferState.InProgress);
    if (contentLength < this.multiThreadThreshold) {
        throw new CosClientException("contentLenth " + contentLength + " < " + this.multiThreadThreshold + " should not use resumabledownload.");
    }
    long start = 0;
    publishProgress(listener, ProgressEventType.TRANSFER_STARTED_EVENT);
    while (contentLength > start) {
        long bytesToRead = Math.min(partSize, contentLength - start);
        long end = start + bytesToRead - 1;
        String block = String.format("%d-%d", start, end);
        if (downloadRecord.hasDownloadedBlocks(block)) {
            log.debug("part found in download record: " + block);
            CRC64 crc64 = new CRC64();
            byte[] buffer = new byte[1024 * 10];
            int readBytes;
            destRandomAccessFile.seek(start);
            while (bytesToRead > 0 && (readBytes = destRandomAccessFile.read(buffer)) != -1) {
                long updateBytes = Math.min(readBytes, bytesToRead);
                bytesToRead -= updateBytes;
                crc64.update(buffer, (int) updateBytes);
            }
            skippedParts.add(new DownloadPart(start, end, crc64.getValue()));
            transferProgress.updateProgress(end + 1 - start);
        } else {
            GetObjectRequest getObj = copyGetObjectRequest(req);
            getObj.setRange(start, end);
            if (threadPool.isShutdown()) {
                publishProgress(listener, ProgressEventType.TRANSFER_CANCELED_EVENT);
                throw new CancellationException("TransferManager has been shutdown");
            }
            futures.add(threadPool.submit(new RangeDownloadCallable(cos, getObj, destFile, destFileChannel, downloadRecord)));
        }
        start = end + 1;
    }
}

Example 2 with CRC64

use of com.qcloud.cos.utils.CRC64 in project hadoop-cos by tencentyun.

the class CosNPosixExtensionDataOutputStream method resumeForWrite.

private void resumeForWrite() throws IOException {
    super.resetContext();
    super.initNewCurrentPartResource();
    FileMetadata fileMetadata = super.nativeStore.retrieveMetadata(super.cosKey);
    if (null == fileMetadata) {
        throw new IOException(String.format("The cos key [%s] is not found.", super.cosKey));
    }
    if (!fileMetadata.isFile()) {
        throw new IOException("The cos key is a directory object. Can not resume the write operation for it.");
    }
    // resume for write operation.
    try {
        if (fileMetadata.getLength() < super.partSize) {
            // Single file resume
            try (InputStream inputStream = super.nativeStore.retrieve(super.cosKey)) {
                byte[] chunk = new byte[(int) (4 * Unit.KB)];
                int readBytes = inputStream.read(chunk);
                while (readBytes != -1) {
                    super.write(chunk, 0, readBytes);
                    readBytes = inputStream.read(chunk);
                }
            }
        } else {
            // Multipart copy resume
            super.multipartUpload = new MultipartUploadEx(super.cosKey);
            long copyRemaining = fileMetadata.getLength();
            long firstByte = 0;
            long lastByte = firstByte + super.partSize - 1;
            while (copyRemaining >= super.partSize) {
                UploadPartCopy uploadPartCopy = new UploadPartCopy(super.cosKey, super.cosKey, super.currentPartNumber++, firstByte, lastByte);
                ((MultipartUploadEx) super.multipartUpload).uploadPartCopyAsync(uploadPartCopy);
                copyRemaining -= ((lastByte - firstByte) + 1);
                firstByte = lastByte + 1;
                lastByte = firstByte + super.partSize - 1;
            }
            // initialize the last part
            if (copyRemaining > 0) {
                lastByte = firstByte + copyRemaining - 1;
                try (InputStream inputStream = super.nativeStore.retrieveBlock(super.cosKey, firstByte, lastByte)) {
                    byte[] chunk = new byte[(int) (4 * Unit.KB)];
                    int readBytes = inputStream.read(chunk);
                    while (readBytes != -1) {
                        super.write(chunk, 0, readBytes);
                        readBytes = inputStream.read(chunk);
                    }
                }
            }
            // initialize the consistency checker.
            BigInteger bigInteger = new BigInteger(fileMetadata.getCrc64ecm());
            this.consistencyChecker = new ConsistencyChecker(super.nativeStore, super.cosKey, new CRC64(bigInteger.longValue()), fileMetadata.getLength());
        }
    } catch (Exception e) {
        LOG.error("Fail to resume for writing. Abort it.", e);
        super.abort();
        throw new IOException(e);
    }
}

Example 3 with CRC64

use of com.qcloud.cos.utils.CRC64 in project cos-java-sdk-v5 by tencentyun.

the class RangeDownloadCallable method call.

public DownloadPart call() throws Exception {
    COSObject object = cos.getObject(request);
    InputStream input = object.getObjectContent();
    ObjectMetadata meta = object.getObjectMetadata();
    long[] range = request.getRange();
    long start = range[0];
    long end = range[1];
    long position = start;
    ByteBuffer tmpBuf = ByteBuffer.allocateDirect(1024 * 1024);
    byte[] buffer = new byte[1024 * 10];
    int bytesRead;
    CRC64 crc64 = new CRC64();
    while ((bytesRead = input.read(buffer)) > -1) {
        start += bytesRead;
        crc64.update(buffer, bytesRead);
        if (tmpBuf.remaining() < bytesRead) {
            tmpBuf.flip();
            position += destFileChannel.write(tmpBuf, position);
            tmpBuf.clear();
        }
        tmpBuf.put(buffer, 0, bytesRead);
    }
    tmpBuf.flip();
    destFileChannel.write(tmpBuf, position);
    if (start != end + 1) {
        destFileChannel.close();
        destFile.delete();
        String msg = String.format("get object want %d bytes, but got %d bytes, reqeust_id: %s", end + 1, start, meta.getRequestId());
        throw new CosClientException(msg);
    }
    String block = String.format("%d-%d", range[0], range[1]);
    downloadRecord.putDownloadedBlocks(block);
    downloadRecord.dump();
    return new DownloadPart(range[0], range[1], crc64.getValue());
}

Example 4 with CRC64

use of com.qcloud.cos.utils.CRC64 in project cos-java-sdk-v5 by tencentyun.

the class ResumableDownloadMonitor method checkCRC.

void checkCRC(List<DownloadPart> downloadParts) throws IOException {
    Collections.sort(downloadParts, new Comparator<DownloadPart>() {

        @Override
        public int compare(DownloadPart part1, DownloadPart part2) {
            return (int) (part1.start - part2.start);
        }
    });
    CRC64 crc64 = new CRC64();
    for (DownloadPart part : downloadParts) {
        crc64 = CRC64.combine(crc64, new CRC64(part.crc64), part.getContentLength());
    }
    long crc64Download = crc64.getValue();
    long crc64Cos = crc64ToLong(downloadRecord.getCrc64ecma());
    log.debug("download crc " + crc64Download + " cos crc " + crc64Cos);
    if (crc64Download != crc64Cos) {
        destFile.delete();
        throw new CosClientException("download file has diff crc64 with cos file, cos: " + crc64Cos + " downloaded: " + crc64Download);
    }
}