Examples with OSS com.aliyun.oss.OSS used on opensource projects

Example 46 with OSS

use of com.aliyun.oss.OSS in project druid by druid-io.

the class OssDataSegmentPusherTest method testPushInternal.

private void testPushInternal(boolean useUniquePath, String matcher) throws Exception {
    OSS client = EasyMock.createStrictMock(OSS.class);
    EasyMock.expect(client.putObject(EasyMock.anyObject())).andReturn(new PutObjectResult()).once();
    EasyMock.replay(client);
    OssStorageConfig config = new OssStorageConfig();
    config.setBucket("bucket");
    config.setPrefix("key");
    OssDataSegmentPusher pusher = new OssDataSegmentPusher(client, config);
    // Create a mock segment on disk
    File tmp = tempFolder.newFile("version.bin");
    final byte[] data = new byte[] { 0x0, 0x0, 0x0, 0x1 };
    Files.write(data, tmp);
    final long size = data.length;
    DataSegment segmentToPush = new DataSegment("foo", Intervals.of("2015/2016"), "0", new HashMap<>(), new ArrayList<>(), new ArrayList<>(), NoneShardSpec.instance(), 0, size);
    DataSegment segment = pusher.push(tempFolder.getRoot(), segmentToPush, useUniquePath);
    Assert.assertEquals(segmentToPush.getSize(), segment.getSize());
    Assert.assertEquals(1, (int) segment.getBinaryVersion());
    Assert.assertEquals("bucket", segment.getLoadSpec().get("bucket"));
    Assert.assertTrue(segment.getLoadSpec().get("key").toString(), Pattern.compile(matcher).matcher(segment.getLoadSpec().get("key").toString()).matches());
    Assert.assertEquals("oss_zip", segment.getLoadSpec().get("type"));
    EasyMock.verify(client);
}

Example 47 with OSS

use of com.aliyun.oss.OSS in project druid by druid-io.

the class OssDataSegmentMover method selfCheckingMove.

/**
 * Copies an object and after that checks that the object is present at the target location, via a separate API call.
 * If it is not, an exception is thrown, and the object is not deleted at the old location. This "paranoic" check
 * is added after it was observed that oss may report a successful move, and the object is not found at the target
 * location.
 */
private void selfCheckingMove(String srcBucket, String dstBucket, String srcPath, String dstPath, String copyMsg) throws IOException, SegmentLoadingException {
    if (srcBucket.equals(dstBucket) && srcPath.equals(dstPath)) {
        log.info("No need to move file[%s://%s/%s] onto itself", OssStorageDruidModule.SCHEME, srcBucket, srcPath);
        return;
    }
    final OSS client = this.clientSupplier.get();
    if (client.doesObjectExist(srcBucket, srcPath)) {
        final ObjectListing listResult = client.listObjects(new ListObjectsRequest(srcBucket, srcPath, null, null, 1));
        // keyCount is still zero.
        if (listResult.getObjectSummaries().size() == 0) {
            // should never happen
            throw new ISE("Unable to list object [%s://%s/%s]", OssStorageDruidModule.SCHEME, srcBucket, srcPath);
        }
        final OSSObjectSummary objectSummary = listResult.getObjectSummaries().get(0);
        if (objectSummary.getStorageClass() != null && objectSummary.getStorageClass().equals(StorageClass.IA.name())) {
            throw new OSSException(StringUtils.format("Cannot move file[%s://%s/%s] of storage class glacier, skipping.", OssStorageDruidModule.SCHEME, srcBucket, srcPath));
        } else {
            log.info("Moving file %s", copyMsg);
            final CopyObjectRequest copyRequest = new CopyObjectRequest(srcBucket, srcPath, dstBucket, dstPath);
            client.copyObject(copyRequest);
            if (!client.doesObjectExist(dstBucket, dstPath)) {
                throw new IOE("After copy was reported as successful the file doesn't exist in the target location [%s]", copyMsg);
            }
            deleteWithRetriesSilent(srcBucket, srcPath);
            log.debug("Finished moving file %s", copyMsg);
        }
    } else {
        // ensure object exists in target location
        if (client.doesObjectExist(dstBucket, dstPath)) {
            log.info("Not moving file [%s://%s/%s], already present in target location [%s://%s/%s]", OssStorageDruidModule.SCHEME, srcBucket, srcPath, OssStorageDruidModule.SCHEME, dstBucket, dstPath);
        } else {
            throw new SegmentLoadingException("Unable to move file %s, not present in either source or target location", copyMsg);
        }
    }
}

Example 48 with OSS

use of com.aliyun.oss.OSS in project druid by druid-io.

the class OssTimestampVersionedDataFinderTest method testFindSelf.

@Test
public void testFindSelf() {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    OSS ossClient = EasyMock.createStrictMock(OSS.class);
    OSSObjectSummary object0 = new OSSObjectSummary();
    object0.setBucketName(bucket);
    object0.setKey(keyPrefix + "/renames-0.gz");
    object0.setLastModified(new Date(0));
    object0.setSize(10);
    final ObjectListing result = new ObjectListing();
    result.getObjectSummaries().add(object0);
    result.setTruncated(false);
    EasyMock.expect(ossClient.listObjects(EasyMock.anyObject(ListObjectsRequest.class))).andReturn(result).once();
    OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(ossClient);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(ossClient);
    URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern);
    EasyMock.verify(ossClient);
    URI expected = URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, object0.getKey()));
    Assert.assertEquals(expected, latest);
}

Example 49 with OSS

use of com.aliyun.oss.OSS in project druid by druid-io.

the class OssTimestampVersionedDataFinderTest method testMissing.

@Test
public void testMissing() {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    OSS oss = EasyMock.createStrictMock(OSS.class);
    final ObjectListing result = new ObjectListing();
    result.setTruncated(false);
    EasyMock.expect(oss.listObjects(EasyMock.anyObject(ListObjectsRequest.class))).andReturn(result).once();
    OssTimestampVersionedDataFinder finder = new OssTimestampVersionedDataFinder(oss);
    Pattern pattern = Pattern.compile("renames-[0-9]*\\.gz");
    EasyMock.replay(oss);
    URI latest = finder.getLatestVersion(URI.create(StringUtils.format("%s://%s/%s", OssStorageDruidModule.SCHEME, bucket, keyPrefix)), pattern);
    EasyMock.verify(oss);
    Assert.assertEquals(null, latest);
}