Examples with DataStorageAction com.epam.pipeline.entity.datastorage.DataStorageAction used on opensource projects

Example 1 with DataStorageAction

use of com.epam.pipeline.entity.datastorage.DataStorageAction in project cloud-pipeline by epam.

the class DataStorageManager method generateTemporaryCredentials.

public AbstractTemporaryCredentials generateTemporaryCredentials(List<DataStorageAction> actions) {
    AbstractDataStorage dataStorage = ListUtils.emptyIfNull(actions).stream().findFirst().map(action -> load(action.getId())).orElseThrow(() -> new IllegalArgumentException("Actions are not provided"));
    Assert.isTrue(dataStorage instanceof S3bucketDataStorage, "Temporary credentials are supported only for S3 Buckets.");
    S3bucketDataStorage bucket = (S3bucketDataStorage) dataStorage;
    AwsRegion awsRegion = awsRegionManager.getAwsRegion(bucket);
    actions.forEach(action -> {
        AbstractDataStorage loaded = action.getId().equals(dataStorage.getId()) ? dataStorage : load(action.getId());
        Assert.isTrue(loaded instanceof S3bucketDataStorage, "Temporary credentials are supported only for S3 Buckets.");
        action.setBucketName(loaded.getPath());
        Long regionId = ((S3bucketDataStorage) loaded).getRegionId();
        Assert.isTrue(Objects.equals(regionId, bucket.getRegionId()), "Actions shall be requested for buckets from the same region");
    });
    return dataStorageFactory.temporaryCredentials(bucket.getType()).withRole(preferenceManager.getPreference(SystemPreferences.DATA_STORAGE_TEMP_CREDENTIALS_ROLE)).withDurationSeconds(preferenceManager.getPreference(SystemPreferences.DATA_STORAGE_TEMP_CREDENTIALS_DURATION)).withKmsArn(awsRegion.getKmsKeyArn()).withRegionId(awsRegion.getAwsRegionName()).generate(actions);
}

Example 2 with DataStorageAction

use of com.epam.pipeline.entity.datastorage.DataStorageAction in project cloud-pipeline by epam.

the class S3Synchronizer method createIndexAndDocuments.

@SuppressWarnings("PMD.AvoidCatchingGenericException")
void createIndexAndDocuments(final AbstractDataStorage dataStorage) {
    EntityPermissionVO entityPermission = cloudPipelineAPIClient.loadPermissionsForEntity(dataStorage.getId(), dataStorage.getAclClass());
    PermissionsContainer permissionsContainer = new PermissionsContainer();
    if (entityPermission != null) {
        permissionsContainer.add(entityPermission.getPermissions(), dataStorage.getOwner());
    }
    String alias = indexPrefix + indexName + String.format("-%d", dataStorage.getId());
    String indexName = generateRandomString(5).toLowerCase() + "-" + alias;
    try {
        String currentIndexName = elasticsearchServiceClient.getIndexNameByAlias(alias);
        elasticIndexService.createIndexIfNotExist(indexName, indexSettingsPath);
        DataStorageAction action = new DataStorageAction();
        action.setBucketName(dataStorage.getPath());
        action.setId(dataStorage.getId());
        action.setWrite(false);
        action.setRead(true);
        AbstractTemporaryCredentials credentials = cloudPipelineAPIClient.generateTemporaryCredentials(Collections.singletonList(action));
        BulkRequestCreator bulkRequestCreator = requests -> elasticsearchServiceClient.sendRequests(indexName, requests);
        S3Helper s3Helper = new S3Helper(enableTags, credentials, bulkRequestCreator, dataStorage, indexName, bulkInsertSize, permissionsContainer);
        s3Helper.addItems();
        elasticsearchServiceClient.createIndexAlias(indexName, alias);
        if (StringUtils.hasText(currentIndexName)) {
            elasticsearchServiceClient.deleteIndex(currentIndexName);
        }
    } catch (Exception e) {
        log.error(e.getMessage(), e);
        if (elasticsearchServiceClient.isIndexExists(indexName)) {
            elasticsearchServiceClient.deleteIndex(indexName);
        }
    }
}

Example 3 with DataStorageAction

use of com.epam.pipeline.entity.datastorage.DataStorageAction in project cloud-pipeline by epam.

the class S3TemporaryCredentials method createPolicyWithPermissions.

private String createPolicyWithPermissions(List<DataStorageAction> actions) {
    ObjectNode resultPolicy = JsonNodeFactory.instance.objectNode();
    resultPolicy.put("Version", "2012-10-17");
    ArrayNode statements = resultPolicy.putArray("Statement");
    final String kmsArn = getKmsArn();
    if (StringUtils.isNotBlank(kmsArn)) {
        addKmsActionToStatement(kmsArn, statements);
    }
    for (DataStorageAction action : actions) {
        addActionToStatement(action, statements, true);
        addActionToStatement(action, statements, false);
    }
    return resultPolicy.toString();
}