Examples with StorageObject com.google.api.services.storage.model.StorageObject used on opensource projects

Example 71 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleCloudStorageInputSource method getPrefixesSplitStream.

@Override
protected Stream<InputSplit<List<CloudObjectLocation>>> getPrefixesSplitStream(@Nonnull SplitHintSpec splitHintSpec) {
    final Iterator<List<StorageObject>> splitIterator = splitHintSpec.split(storageObjectIterable().iterator(), storageObject -> {
        final BigInteger sizeInBigInteger = storageObject.getSize();
        long sizeInLong;
        if (sizeInBigInteger == null) {
            sizeInLong = Long.MAX_VALUE;
        } else {
            try {
                sizeInLong = sizeInBigInteger.longValueExact();
            } catch (ArithmeticException e) {
                LOG.warn(e, "The object [%s, %s] has a size [%s] out of the range of the long type. " + "The max long value will be used for its size instead.", storageObject.getBucket(), storageObject.getName(), sizeInBigInteger);
                sizeInLong = Long.MAX_VALUE;
            }
        }
        return new InputFileAttribute(sizeInLong);
    });
    return Streams.sequentialStreamFrom(splitIterator).map(objects -> objects.stream().map(this::byteSourceFromStorageObject).collect(Collectors.toList())).map(InputSplit::new);
}

Example 72 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTaskLogsTest method test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs.

@Test
public void test_killAll_recoverableExceptionWhenDeletingObjects_deletesAllTaskLogs() throws IOException {
    StorageObject object1 = GoogleTestUtils.newStorageObject(BUCKET, KEY_1, TIME_0);
    EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
    Storage.Objects.List listRequest = GoogleTestUtils.expectListRequest(storage, PREFIX_URI);
    GoogleTestUtils.expectListObjects(listRequest, PREFIX_URI, MAX_KEYS, ImmutableList.of(object1));
    GoogleTestUtils.expectDeleteObjects(storage, ImmutableList.of(object1), ImmutableMap.of(object1, RECOVERABLE_EXCEPTION));
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.replay(listRequest, inputDataConfig, storage, timeSupplier);
    googleTaskLogs.killAll();
    EasyMock.verify(listRequest, inputDataConfig, storage, timeSupplier);
}

Example 73 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTaskLogsTest method test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAllTaskLogs.

@Test
public void test_killOlderThan_recoverableExceptionWhenListingObjects_deletesAllTaskLogs() throws IOException {
    StorageObject object1 = GoogleTestUtils.newStorageObject(BUCKET, KEY_1, TIME_0);
    Storage.Objects.List listRequest = GoogleTestUtils.expectListRequest(storage, PREFIX_URI);
    GoogleTestUtils.expectListObjects(listRequest, PREFIX_URI, MAX_KEYS, ImmutableList.of(object1));
    GoogleTestUtils.expectDeleteObjects(storage, ImmutableList.of(object1), ImmutableMap.of(object1, RECOVERABLE_EXCEPTION));
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.replay(listRequest, inputDataConfig, storage);
    googleTaskLogs.killOlderThan(TIME_NOW);
    EasyMock.verify(listRequest, inputDataConfig, storage);
}

Example 74 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTaskLogsTest method test_killAll_noException_deletesAllTaskLogs.

@Test
public void test_killAll_noException_deletesAllTaskLogs() throws IOException {
    StorageObject object1 = GoogleTestUtils.newStorageObject(BUCKET, KEY_1, TIME_0);
    StorageObject object2 = GoogleTestUtils.newStorageObject(BUCKET, KEY_2, TIME_1);
    EasyMock.expect(timeSupplier.getAsLong()).andReturn(TIME_NOW);
    Storage.Objects.List listRequest = GoogleTestUtils.expectListRequest(storage, PREFIX_URI);
    GoogleTestUtils.expectListObjects(listRequest, PREFIX_URI, MAX_KEYS, ImmutableList.of(object1, object2));
    GoogleTestUtils.expectDeleteObjects(storage, ImmutableList.of(object1, object2), ImmutableMap.of());
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.replay(listRequest, inputDataConfig, storage, timeSupplier);
    googleTaskLogs.killAll();
    EasyMock.verify(listRequest, inputDataConfig, storage, timeSupplier);
}

Example 75 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTimestampVersionedDataFinderTest method getLatestVersion.

@Test
public void getLatestVersion() {
    String bucket = "bucket";
    String keyPrefix = "prefix/dir/0";
    // object for directory prefix/dir/0/
    final StorageObject storageObject1 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "//", 0);
    storageObject1.setUpdated(new DateTime(System.currentTimeMillis()));
    final StorageObject storageObject2 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v1", 1);
    storageObject2.setUpdated(new DateTime(System.currentTimeMillis()));
    final StorageObject storageObject3 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/v2", 1);
    storageObject3.setUpdated(new DateTime(System.currentTimeMillis() + 100));
    final StorageObject storageObject4 = ObjectStorageIteratorTest.makeStorageObject(bucket, keyPrefix + "/other", 4);
    storageObject4.setUpdated(new DateTime(System.currentTimeMillis() + 100));
    final GoogleStorage storage = ObjectStorageIteratorTest.makeMockClient(ImmutableList.of(storageObject1, storageObject2, storageObject3, storageObject4));
    final GoogleTimestampVersionedDataFinder finder = new GoogleTimestampVersionedDataFinder(storage);
    Pattern pattern = Pattern.compile("v.*");
    URI latest = finder.getLatestVersion(URI.create(StringUtils.format("gs://%s/%s", bucket, keyPrefix)), pattern);
    URI expected = URI.create(StringUtils.format("gs://%s/%s", bucket, storageObject3.getName()));
    Assert.assertEquals(expected, latest);
}