Examples with StorageUnitConverter org.apache.hop.core.util.StorageUnitConverter used on opensource projects

Example 1 with StorageUnitConverter

use of org.apache.hop.core.util.StorageUnitConverter in project hop by apache.

the class S3FileObjectTest method setUp.

@Before
public void setUp() throws Exception {
    s3ServiceMock = mock(AmazonS3.class);
    S3Object s3Object = new S3Object();
    s3Object.setKey(OBJECT_NAME);
    s3Object.setBucketName(BUCKET_NAME);
    filename = new S3FileName(SCHEME, BUCKET_NAME, BUCKET_NAME, FileType.FOLDER);
    S3FileName rootFileName = new S3FileName(SCHEME, "", "", FileType.FOLDER);
    S3HopProperty s3HopProperty = mock(S3HopProperty.class);
    when(s3HopProperty.getPartSize()).thenReturn("5MB");
    S3FileSystem fileSystem = new S3FileSystem(rootFileName, new FileSystemOptions(), new StorageUnitConverter(), s3HopProperty);
    fileSystemSpy = spy(fileSystem);
    VfsComponentContext context = mock(VfsComponentContext.class);
    final DefaultFileSystemManager fsm = new DefaultFileSystemManager();
    FilesCache cache = mock(FilesCache.class);
    fsm.setFilesCache(cache);
    fsm.setCacheStrategy(CacheStrategy.ON_RESOLVE);
    when(context.getFileSystemManager()).thenReturn(fsm);
    fileSystemSpy.setContext(context);
    S3FileObject s3FileObject = new S3FileObject(filename, fileSystemSpy);
    s3FileObjectBucketSpy = spy(s3FileObject);
    s3FileObjectFileSpy = spy(new S3FileObject(new S3FileName(SCHEME, BUCKET_NAME, BUCKET_NAME + "/" + origKey, FileType.IMAGINARY), fileSystemSpy));
    S3FileObject s3FileObjectRoot = new S3FileObject(rootFileName, fileSystemSpy);
    s3FileObjectSpyRoot = spy(s3FileObjectRoot);
    // specify the behaviour of S3 Service
    // when( s3ServiceMock.getBucket( BUCKET_NAME ) ).thenReturn( testBucket );
    when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
    when(s3ServiceMock.getObject(BUCKET_NAME, OBJECT_NAME)).thenReturn(s3Object);
    when(s3ServiceMock.listBuckets()).thenReturn(createBuckets());
    when(s3ServiceMock.doesBucketExistV2(BUCKET_NAME)).thenReturn(true);
    childObjectListing = mock(ObjectListing.class);
    when(childObjectListing.getObjectSummaries()).thenReturn(createObjectSummaries(0)).thenReturn(new ArrayList<>());
    when(childObjectListing.getCommonPrefixes()).thenReturn(new ArrayList<>()).thenReturn(createCommonPrefixes(3));
    when(childObjectListing.isTruncated()).thenReturn(true).thenReturn(false);
    when(s3ServiceMock.listObjects(any(ListObjectsRequest.class))).thenReturn(childObjectListing);
    when(s3ServiceMock.listObjects(anyString(), anyString())).thenReturn(childObjectListing);
    when(s3ServiceMock.listNextBatchOfObjects(any(ObjectListing.class))).thenReturn(childObjectListing);
    s3ObjectMock = mock(S3Object.class);
    s3ObjectInputStream = mock(S3ObjectInputStream.class);
    s3ObjectMetadata = mock(ObjectMetadata.class);
    when(s3ObjectMock.getObjectContent()).thenReturn(s3ObjectInputStream);
    when(s3ServiceMock.getObjectMetadata(anyString(), anyString())).thenReturn(s3ObjectMetadata);
    when(s3ObjectMetadata.getContentLength()).thenReturn(contentLength);
    when(s3ObjectMetadata.getLastModified()).thenReturn(testDate);
    when(s3ServiceMock.getObject(anyString(), anyString())).thenReturn(s3ObjectMock);
    when(fileSystemSpy.getS3Client()).thenReturn(s3ServiceMock);
}

Example 2 with StorageUnitConverter

use of org.apache.hop.core.util.StorageUnitConverter in project hop by apache.

the class S3FileSystemTest method testParsePartSize.

@Test
public void testParsePartSize() throws Exception {
    S3FileSystem s3FileSystem = getTestInstance();
    s3FileSystem.storageUnitConverter = new StorageUnitConverter();
    long _5MBLong = 5L * 1024L * 1024L;
    long _124MBLong = 124L * 1024L * 1024L;
    long _5GBLong = 5L * 1024L * 1024L * 1024L;
    long _12GBLong = 12L * 1024L * 1024L * 1024L;
    long minimumPartSize = _5MBLong;
    long maximumPartSize = _5GBLong;
    // TEST 1: below minimum
    assertEquals(minimumPartSize, s3FileSystem.parsePartSize("1MB"));
    // TEST 2: at minimum
    assertEquals(minimumPartSize, s3FileSystem.parsePartSize("5MB"));
    // TEST 3: between minimum and maximum
    assertEquals(_124MBLong, s3FileSystem.parsePartSize("124MB"));
    // TEST 4: at maximum
    assertEquals(maximumPartSize, s3FileSystem.parsePartSize("5GB"));
    // TEST 5: above maximum
    assertEquals(_12GBLong, s3FileSystem.parsePartSize("12GB"));
}

Example 3 with StorageUnitConverter

use of org.apache.hop.core.util.StorageUnitConverter in project hop by apache.

the class S3FileSystemTest method testConvertToLong.

@Test
public void testConvertToLong() throws Exception {
    S3FileSystem s3FileSystem = getTestInstance();
    long _10MBLong = 10L * 1024L * 1024L;
    s3FileSystem.storageUnitConverter = new StorageUnitConverter();
    assertEquals(_10MBLong, s3FileSystem.convertToLong("10MB"));
}

Example 4 with StorageUnitConverter

use of org.apache.hop.core.util.StorageUnitConverter in project hop by apache.

the class S3FileSystemTest method getPartSize.

@Test
public void getPartSize() throws Exception {
    S3FileSystem s3FileSystem = getTestInstance();
    s3FileSystem.storageUnitConverter = new StorageUnitConverter();
    S3HopProperty s3HopProperty = mock(S3HopProperty.class);
    when(s3HopProperty.getPartSize()).thenReturn("6MB");
    s3FileSystem.s3HopProperty = s3HopProperty;
    // TEST 1: Below max
    assertEquals(6 * 1024 * 1024, s3FileSystem.getPartSize());
    // TEst 2: above max
    when(s3HopProperty.getPartSize()).thenReturn("600GB");
    assertEquals(Integer.MAX_VALUE, s3FileSystem.getPartSize());
}