Search in sources :

Example 1 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project beam by apache.

the class GcsUtilTest method testGCSChannelCloseIdempotent.

@Test
public void testGCSChannelCloseIdempotent() throws IOException {
    SeekableByteChannel channel = new GoogleCloudStorageReadChannel(null, "dummybucket", "dummyobject", null, new ClientRequestHelper<StorageObject>());
    channel.close();
    channel.close();
}
Also used : SeekableByteChannel(java.nio.channels.SeekableByteChannel) GoogleCloudStorageReadChannel(com.google.cloud.hadoop.gcsio.GoogleCloudStorageReadChannel) StorageObject(com.google.api.services.storage.model.StorageObject) Test(org.junit.Test)

Example 2 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project ignite by apache.

the class TcpDiscoveryGoogleStorageIpFinder method registerAddresses.

/**
 * {@inheritDoc}
 */
@Override
public void registerAddresses(Collection<InetSocketAddress> addrs) throws IgniteSpiException {
    assert !F.isEmpty(addrs);
    init();
    for (InetSocketAddress addr : addrs) {
        String key = keyFromAddr(addr);
        StorageObject object = new StorageObject();
        object.setBucket(bucketName);
        object.setName(key);
        InputStreamContent content = new InputStreamContent("application/octet-stream", OBJECT_CONTENT);
        content.setLength(OBJECT_CONTENT.available());
        try {
            Storage.Objects.Insert insertObject = storage.objects().insert(bucketName, object, content);
            insertObject.execute();
        } catch (Exception e) {
            throw new IgniteSpiException("Failed to put entry [bucketName=" + bucketName + ", entry=" + key + ']', e);
        }
    }
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) InetSocketAddress(java.net.InetSocketAddress) InputStreamContent(com.google.api.client.http.InputStreamContent) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) IgniteSpiException(org.apache.ignite.spi.IgniteSpiException) GoogleJsonResponseException(com.google.api.client.googleapis.json.GoogleJsonResponseException) GeneralSecurityException(java.security.GeneralSecurityException) IgniteInterruptedCheckedException(org.apache.ignite.internal.IgniteInterruptedCheckedException) IOException(java.io.IOException)

Example 3 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class BlobInfo method toPb.

StorageObject toPb() {
    StorageObject storageObject = blobId.toPb();
    if (acl != null) {
        storageObject.setAcl(Lists.transform(acl, new Function<Acl, ObjectAccessControl>() {

            @Override
            public ObjectAccessControl apply(Acl acl) {
                return acl.toObjectPb();
            }
        }));
    }
    if (deleteTime != null) {
        storageObject.setTimeDeleted(new DateTime(deleteTime));
    }
    if (updateTime != null) {
        storageObject.setUpdated(new DateTime(updateTime));
    }
    if (createTime != null) {
        storageObject.setTimeCreated(new DateTime(createTime));
    }
    if (size != null) {
        storageObject.setSize(BigInteger.valueOf(size));
    }
    if (owner != null) {
        storageObject.setOwner(new Owner().setEntity(owner.toPb()));
    }
    if (storageClass != null) {
        storageObject.setStorageClass(storageClass.toString());
    }
    Map<String, String> pbMetadata = metadata;
    if (metadata != null && !Data.isNull(metadata)) {
        pbMetadata = Maps.newHashMapWithExpectedSize(metadata.size());
        for (Map.Entry<String, String> entry : metadata.entrySet()) {
            pbMetadata.put(entry.getKey(), firstNonNull(entry.getValue(), Data.<String>nullOf(String.class)));
        }
    }
    if (customerEncryption != null) {
        storageObject.setCustomerEncryption(customerEncryption.toPb());
    }
    storageObject.setMetadata(pbMetadata);
    storageObject.setCacheControl(cacheControl);
    storageObject.setContentEncoding(contentEncoding);
    storageObject.setCrc32c(crc32c);
    storageObject.setContentType(contentType);
    storageObject.setMd5Hash(md5);
    storageObject.setMediaLink(mediaLink);
    storageObject.setMetageneration(metageneration);
    storageObject.setContentDisposition(contentDisposition);
    storageObject.setComponentCount(componentCount);
    storageObject.setContentLanguage(contentLanguage);
    storageObject.setEtag(etag);
    storageObject.setId(generatedId);
    storageObject.setSelfLink(selfLink);
    return storageObject;
}
Also used : Function(com.google.common.base.Function) Owner(com.google.api.services.storage.model.StorageObject.Owner) StorageObject(com.google.api.services.storage.model.StorageObject) HashMap(java.util.HashMap) AbstractMap(java.util.AbstractMap) Map(java.util.Map) DateTime(com.google.api.client.util.DateTime)

Example 4 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class HttpStorageRpc method open.

@Override
public String open(StorageObject object, Map<Option, ?> options) {
    try {
        Insert req = storage.objects().insert(object.getBucket(), object);
        GenericUrl url = req.buildHttpRequest().getUrl();
        String scheme = url.getScheme();
        String host = url.getHost();
        String path = "/upload" + url.getRawPath();
        url = new GenericUrl(scheme + "://" + host + path);
        url.set("uploadType", "resumable");
        url.set("name", object.getName());
        for (Option option : options.keySet()) {
            Object content = option.get(options);
            if (content != null) {
                url.set(option.value(), content.toString());
            }
        }
        JsonFactory jsonFactory = storage.getJsonFactory();
        HttpRequestFactory requestFactory = storage.getRequestFactory();
        HttpRequest httpRequest = requestFactory.buildPostRequest(url, new JsonHttpContent(jsonFactory, object));
        HttpHeaders requestHeaders = httpRequest.getHeaders();
        requestHeaders.set("X-Upload-Content-Type", firstNonNull(object.getContentType(), "application/octet-stream"));
        String key = Option.CUSTOMER_SUPPLIED_KEY.getString(options);
        if (key != null) {
            BaseEncoding base64 = BaseEncoding.base64();
            HashFunction hashFunction = Hashing.sha256();
            requestHeaders.set("x-goog-encryption-algorithm", "AES256");
            requestHeaders.set("x-goog-encryption-key", key);
            requestHeaders.set("x-goog-encryption-key-sha256", base64.encode(hashFunction.hashBytes(base64.decode(key)).asBytes()));
        }
        HttpResponse response = httpRequest.execute();
        if (response.getStatusCode() != 200) {
            GoogleJsonError error = new GoogleJsonError();
            error.setCode(response.getStatusCode());
            error.setMessage(response.getStatusMessage());
            throw translate(error);
        }
        return response.getHeaders().getLocation();
    } catch (IOException ex) {
        throw translate(ex);
    }
}
Also used : HttpRequest(com.google.api.client.http.HttpRequest) HttpHeaders(com.google.api.client.http.HttpHeaders) HttpRequestFactory(com.google.api.client.http.HttpRequestFactory) JsonFactory(com.google.api.client.json.JsonFactory) HttpResponse(com.google.api.client.http.HttpResponse) LowLevelHttpResponse(com.google.api.client.http.LowLevelHttpResponse) GenericUrl(com.google.api.client.http.GenericUrl) JsonHttpContent(com.google.api.client.http.json.JsonHttpContent) IOException(java.io.IOException) Insert(com.google.api.services.storage.Storage.Objects.Insert) BaseEncoding(com.google.common.io.BaseEncoding) HashFunction(com.google.common.hash.HashFunction) StorageObject(com.google.api.services.storage.model.StorageObject) GoogleJsonError(com.google.api.client.googleapis.json.GoogleJsonError)

Example 5 with StorageObject

use of com.google.api.services.storage.model.StorageObject in project google-cloud-java by GoogleCloudPlatform.

the class HttpStorageRpc method list.

@Override
public Tuple<String, Iterable<StorageObject>> list(final String bucket, Map<Option, ?> options) {
    try {
        Objects objects = storage.objects().list(bucket).setProjection(DEFAULT_PROJECTION).setVersions(Option.VERSIONS.getBoolean(options)).setDelimiter(Option.DELIMITER.getString(options)).setPrefix(Option.PREFIX.getString(options)).setMaxResults(Option.MAX_RESULTS.getLong(options)).setPageToken(Option.PAGE_TOKEN.getString(options)).setFields(Option.FIELDS.getString(options)).execute();
        Iterable<StorageObject> storageObjects = Iterables.concat(firstNonNull(objects.getItems(), ImmutableList.<StorageObject>of()), objects.getPrefixes() != null ? Lists.transform(objects.getPrefixes(), objectFromPrefix(bucket)) : ImmutableList.<StorageObject>of());
        return Tuple.of(objects.getNextPageToken(), storageObjects);
    } catch (IOException ex) {
        throw translate(ex);
    }
}
Also used : StorageObject(com.google.api.services.storage.model.StorageObject) Objects(com.google.api.services.storage.model.Objects) IOException(java.io.IOException)

Aggregations

StorageObject (com.google.api.services.storage.model.StorageObject)81 Test (org.junit.Test)33 Objects (com.google.api.services.storage.model.Objects)21 IOException (java.io.IOException)20 ArrayList (java.util.ArrayList)16 Storage (com.google.api.services.storage.Storage)12 List (java.util.List)9 URI (java.net.URI)8 RetryHelperException (com.google.cloud.RetryHelper.RetryHelperException)7 GcsOptions (org.apache.beam.sdk.extensions.gcp.options.GcsOptions)7 GcsPath (org.apache.beam.sdk.extensions.gcp.util.gcsfs.GcsPath)7 GoogleJsonError (com.google.api.client.googleapis.json.GoogleJsonError)6 Nullable (javax.annotation.Nullable)6 File (java.io.File)5 Pattern (java.util.regex.Pattern)5 DateTime (com.google.api.client.util.DateTime)4 BlobSourceOption (com.google.cloud.storage.Storage.BlobSourceOption)4 BlobTargetOption (com.google.cloud.storage.Storage.BlobTargetOption)4 BlobWriteOption (com.google.cloud.storage.Storage.BlobWriteOption)4 BucketSourceOption (com.google.cloud.storage.Storage.BucketSourceOption)4