Examples with StorageObject com.google.api.services.storage.model.StorageObject used on opensource projects

Example 66 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project gradle by gradle.

the class GcsResourceConnector method getMetaData.

@Nullable
@Override
public ExternalResourceMetaData getMetaData(ExternalResourceName location, boolean revalidate) throws ResourceException {
    LOGGER.debug("Attempting to get resource metadata: {}", location);
    StorageObject gcsObject = gcsClient.getResource(location.getUri());
    if (gcsObject == null) {
        return null;
    }
    return toExternalResourceMetaData(location.getUri(), gcsObject);
}

Example 67 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTestUtils method newStorageObject.

public static StorageObject newStorageObject(String bucket, String key, long lastModifiedTimestamp) {
    StorageObject object = new StorageObject();
    object.setBucket(bucket);
    object.setName(key);
    object.setUpdated(new DateTime(lastModifiedTimestamp));
    object.setEtag("etag");
    object.setSize(BigInteger.valueOf(CONTENT.length));
    return object;
}

Example 68 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleTimestampVersionedDataFinder method getLatestVersion.

@Override
public URI getLatestVersion(URI descriptorBase, @Nullable Pattern pattern) {
    try {
        long mostRecent = Long.MIN_VALUE;
        URI latest = null;
        final CloudObjectLocation baseLocation = new CloudObjectLocation(descriptorBase);
        final Objects objects = storage.list(baseLocation.getBucket()).setPrefix(baseLocation.getPath()).setMaxResults(MAX_LISTING_KEYS).execute();
        for (StorageObject storageObject : objects.getItems()) {
            if (GoogleUtils.isDirectoryPlaceholder(storageObject)) {
                continue;
            }
            // remove path prefix from file name
            final CloudObjectLocation objectLocation = new CloudObjectLocation(storageObject.getBucket(), storageObject.getName());
            final String keyString = StringUtils.maybeRemoveLeadingSlash(storageObject.getName().substring(baseLocation.getPath().length()));
            if (pattern != null && !pattern.matcher(keyString).matches()) {
                continue;
            }
            final long latestModified = storageObject.getUpdated().getValue();
            if (latestModified >= mostRecent) {
                mostRecent = latestModified;
                latest = objectLocation.toUri(GoogleStorageDruidModule.SCHEME_GS);
            }
        }
        return latest;
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
}

Example 69 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleDataSegmentKillerTest method test_killAll_noException_deletesAllTaskLogs.

@Test
public void test_killAll_noException_deletesAllTaskLogs() throws IOException {
    StorageObject object1 = GoogleTestUtils.newStorageObject(BUCKET, KEY_1, TIME_0);
    StorageObject object2 = GoogleTestUtils.newStorageObject(BUCKET, KEY_2, TIME_1);
    Storage.Objects.List listRequest = GoogleTestUtils.expectListRequest(storage, PREFIX_URI);
    GoogleTestUtils.expectListObjects(listRequest, PREFIX_URI, MAX_KEYS, ImmutableList.of(object1, object2));
    GoogleTestUtils.expectDeleteObjects(storage, ImmutableList.of(object1, object2), ImmutableMap.of());
    EasyMock.expect(accountConfig.getBucket()).andReturn(BUCKET).anyTimes();
    EasyMock.expect(accountConfig.getPrefix()).andReturn(PREFIX).anyTimes();
    EasyMock.expect(inputDataConfig.getMaxListingLength()).andReturn(MAX_KEYS);
    EasyMock.replay(listRequest, accountConfig, inputDataConfig, storage);
    GoogleDataSegmentKiller killer = new GoogleDataSegmentKiller(storage, accountConfig, inputDataConfig);
    killer.killAll();
    EasyMock.verify(listRequest, accountConfig, inputDataConfig, storage);
}

Example 70 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project druid by druid-io.

the class GoogleCloudStorageInputSourceTest method addExpectedPrefixObjects.

private static void addExpectedPrefixObjects(URI prefix, List<URI> uris) throws IOException {
    final String bucket = prefix.getAuthority();
    Storage.Objects.List listRequest = EasyMock.createMock(Storage.Objects.List.class);
    EasyMock.expect(STORAGE.list(EasyMock.eq(bucket))).andReturn(listRequest).once();
    EasyMock.expect(listRequest.setPageToken(EasyMock.anyString())).andReturn(listRequest).once();
    EasyMock.expect(listRequest.setMaxResults((long) MAX_LISTING_LENGTH)).andReturn(listRequest).once();
    EasyMock.expect(listRequest.setPrefix(EasyMock.eq(StringUtils.maybeRemoveLeadingSlash(prefix.getPath())))).andReturn(listRequest).once();
    List<StorageObject> mockObjects = new ArrayList<>();
    for (URI uri : uris) {
        StorageObject s = new StorageObject();
        s.setBucket(bucket);
        s.setName(uri.getPath());
        s.setSize(BigInteger.valueOf(CONTENT.length));
        mockObjects.add(s);
    }
    Objects response = new Objects();
    response.setItems(mockObjects);
    EasyMock.expect(listRequest.execute()).andReturn(response).once();
    EasyMock.expect(response.getItems()).andReturn(mockObjects).once();
    EasyMock.replay(listRequest);
}