Examples with ExternalStorageLocation com.netflix.conductor.common.run.ExternalStorageLocation used on opensource projects

Example 1 with ExternalStorageLocation

use of com.netflix.conductor.common.run.ExternalStorageLocation in project conductor by Netflix.

the class ClientBase method downloadFromExternalStorage.

/**
 * Uses the {@link PayloadStorage} for downloading large payloads to be used by the client.
 * Gets the uri of the payload fom the server and then downloads from this location.
 *
 * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be downloaded
 * @param path        the relative of the payload in external storage
 * @return the payload object that is stored in external storage
 */
@SuppressWarnings("unchecked")
protected Map<String, Object> downloadFromExternalStorage(ExternalPayloadStorage.PayloadType payloadType, String path) {
    Preconditions.checkArgument(StringUtils.isNotBlank(path), "uri cannot be blank");
    ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.READ, payloadType, path);
    try (InputStream inputStream = payloadStorage.download(externalStorageLocation.getUri())) {
        return objectMapper.readValue(inputStream, Map.class);
    } catch (IOException e) {
        String errorMsg = String.format("Unable to download payload from external storage location: %s", path);
        logger.error(errorMsg, e);
        throw new ConductorClientException(errorMsg, e);
    }
}

Example 2 with ExternalStorageLocation

use of com.netflix.conductor.common.run.ExternalStorageLocation in project conductor by Netflix.

the class AzureBlobPayloadStorage method getLocation.

/**
 * @param operation   the type of {@link Operation} to be performed
 * @param payloadType the {@link PayloadType} that is being accessed
 * @return a {@link ExternalStorageLocation} object which contains the pre-signed URL and the azure blob name
 *  for the json payload
 */
@Override
public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) {
    try {
        ExternalStorageLocation externalStorageLocation = new ExternalStorageLocation();
        String objectKey;
        if (StringUtils.isNotBlank(path)) {
            objectKey = path;
        } else {
            objectKey = getObjectKey(payloadType);
        }
        externalStorageLocation.setPath(objectKey);
        BlockBlobClient blockBlobClient = blobContainerClient.getBlobClient(objectKey).getBlockBlobClient();
        String blobUrl = Utility.urlDecode(blockBlobClient.getBlobUrl());
        if (sasTokenCredential != null) {
            blobUrl = blobUrl + "?" + sasTokenCredential.getSasToken();
        } else {
            BlobSasPermission blobSASPermission = new BlobSasPermission();
            if (operation.equals(Operation.READ)) {
                blobSASPermission.setReadPermission(true);
            } else if (operation.equals(Operation.WRITE)) {
                blobSASPermission.setWritePermission(true);
                blobSASPermission.setCreatePermission(true);
            }
            BlobServiceSasSignatureValues blobServiceSasSignatureValues = new BlobServiceSasSignatureValues(OffsetDateTime.now(ZoneOffset.UTC).plusSeconds(expirationSec), blobSASPermission);
            blobUrl = blobUrl + "?" + blockBlobClient.generateSas(blobServiceSasSignatureValues);
        }
        externalStorageLocation.setUri(blobUrl);
        return externalStorageLocation;
    } catch (BlobStorageException e) {
        String msg = "Error communicating with Azure";
        logger.error(msg, e);
        throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e);
    }
}

Example 3 with ExternalStorageLocation

use of com.netflix.conductor.common.run.ExternalStorageLocation in project conductor by Netflix.

the class AzureBlobPayloadStorageTest method testGetLocation.

private void testGetLocation(AzureBlobPayloadStorage azureBlobPayloadStorage, ExternalPayloadStorage.Operation operation, ExternalPayloadStorage.PayloadType payloadType, String expectedPath) {
    ExternalStorageLocation externalStorageLocation = azureBlobPayloadStorage.getLocation(operation, payloadType, null);
    assertNotNull(externalStorageLocation);
    assertNotNull(externalStorageLocation.getPath());
    assertTrue(externalStorageLocation.getPath().startsWith(expectedPath));
    assertNotNull(externalStorageLocation.getUri());
    assertTrue(externalStorageLocation.getUri().contains(expectedPath));
}

Example 4 with ExternalStorageLocation

use of com.netflix.conductor.common.run.ExternalStorageLocation in project conductor by Netflix.

the class ExternalPayloadStorageUtils method uploadHelper.

@VisibleForTesting
String uploadHelper(byte[] payloadBytes, long payloadSize, ExternalPayloadStorage.PayloadType payloadType) {
    ExternalStorageLocation location = externalPayloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, "");
    externalPayloadStorage.upload(location.getPath(), new ByteArrayInputStream(payloadBytes), payloadSize);
    return location.getPath();
}

Example 5 with ExternalStorageLocation

use of com.netflix.conductor.common.run.ExternalStorageLocation in project conductor by Netflix.

the class S3PayloadStorage method getLocation.

/**
 * @param operation   the type of {@link Operation} to be performed
 * @param payloadType the {@link PayloadType} that is being accessed
 * @return a {@link ExternalStorageLocation} object which contains the pre-signed URL and the s3 object key for the json payload
 */
@Override
public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) {
    try {
        ExternalStorageLocation externalStorageLocation = new ExternalStorageLocation();
        Date expiration = new Date();
        long expTimeMillis = expiration.getTime() + 1000 * expirationSec;
        expiration.setTime(expTimeMillis);
        HttpMethod httpMethod = HttpMethod.GET;
        if (operation == Operation.WRITE) {
            httpMethod = HttpMethod.PUT;
        }
        String objectKey;
        if (StringUtils.isNotBlank(path)) {
            objectKey = path;
        } else {
            objectKey = getObjectKey(payloadType);
        }
        externalStorageLocation.setPath(objectKey);
        GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey).withMethod(httpMethod).withExpiration(expiration);
        externalStorageLocation.setUri(s3Client.generatePresignedUrl(generatePresignedUrlRequest).toURI().toASCIIString());
        return externalStorageLocation;
    } catch (SdkClientException e) {
        String msg = "Error communicating with S3";
        logger.error(msg, e);
        throw new ApplicationException(ApplicationException.Code.BACKEND_ERROR, msg, e);
    } catch (URISyntaxException e) {
        String msg = "Invalid URI Syntax";
        logger.error(msg, e);
        throw new ApplicationException(ApplicationException.Code.INTERNAL_ERROR, msg, e);
    }
}