Examples with S3MultipartUploadService ch.cyberduck.core.s3.S3MultipartUploadService used on opensource projects

Example 1 with S3MultipartUploadService

use of ch.cyberduck.core.s3.S3MultipartUploadService in project cyberduck by iterate-ch.

the class S3MultipartUploadServiceTest method testUploadSinglePart.

@Test
public void testUploadSinglePart() throws Exception {
    // 5L * 1024L * 1024L
    final Path home = new Path("test-eu-central-1-cyberduck", EnumSet.of(Path.Type.volume, Path.Type.directory));
    final Path vault = new Path(home, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory));
    final Path test = new Path(vault, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final CryptoVault cryptomator = new CryptoVault(vault);
    cryptomator.create(session, new VaultCredentials("test"), new DisabledPasswordStore(), vaultVersion);
    session.withRegistry(new DefaultVaultRegistry(new DisabledPasswordStore(), new DisabledPasswordCallback(), cryptomator));
    final CryptoUploadFeature m = new CryptoUploadFeature<>(session, new S3MultipartUploadService(session, new S3WriteFeature(session), 5L * 1024L * 1024L, 5), new S3WriteFeature(session), cryptomator);
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    final int length = 5242880;
    final byte[] content = RandomUtils.nextBytes(length);
    IOUtils.write(content, local.getOutputStream(false));
    final TransferStatus writeStatus = new TransferStatus();
    final FileHeader header = cryptomator.getFileHeaderCryptor().create();
    writeStatus.setHeader(cryptomator.getFileHeaderCryptor().encryptHeader(header));
    writeStatus.setLength(content.length);
    final BytecountStreamListener count = new BytecountStreamListener();
    m.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), count, writeStatus, null);
    assertEquals(content.length, count.getSent());
    assertTrue(writeStatus.isComplete());
    assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    final TransferStatus readStatus = new TransferStatus().withLength(content.length);
    final InputStream in = new CryptoReadFeature(session, new S3ReadFeature(session), cryptomator).read(test, readStatus, new DisabledConnectionCallback());
    new StreamCopier(readStatus, readStatus).transfer(in, buffer);
    assertArrayEquals(content, buffer.toByteArray());
    cryptomator.getFeature(session, Delete.class, new S3DefaultDeleteFeature(session)).delete(Arrays.asList(test, vault), new DisabledLoginCallback(), new Delete.DisabledCallback());
    local.delete();
}

Example 2 with S3MultipartUploadService

use of ch.cyberduck.core.s3.S3MultipartUploadService in project cyberduck by iterate-ch.

the class S3SingleTransferWorkerTest method testTransferredSizeRepeat.

@Test
public void testTransferredSizeRepeat() throws Exception {
    final Local local = new Local(System.getProperty("java.io.tmpdir"), new AlphanumericRandomStringService().random());
    // Minimum multipart upload size
    final byte[] content = new byte[6 * 1024 * 1024];
    new Random().nextBytes(content);
    final OutputStream out = local.getOutputStream(false);
    IOUtils.write(content, out);
    out.close();
    final ProtocolFactory factory = new ProtocolFactory(new HashSet<>(Collections.singleton(new S3Protocol())));
    final Profile profile = new ProfilePlistReader(factory).read(this.getClass().getResourceAsStream("/S3 (HTTPS).cyberduckprofile"));
    final Host host = new Host(profile, profile.getDefaultHostname(), new Credentials(System.getProperties().getProperty("s3.key"), System.getProperties().getProperty("s3.secret")));
    final AtomicBoolean failed = new AtomicBoolean();
    final S3Session session = new S3Session(host, new DefaultX509TrustManager(), new DefaultX509KeyManager()) {

        @Override
        @SuppressWarnings("unchecked")
        public <T> T _getFeature(final Class<T> type) {
            if (type == Upload.class) {
                return (T) new S3MultipartUploadService(this, new S3WriteFeature(this), 5 * 1024L * 1024L, 5) {

                    @Override
                    protected InputStream decorate(final InputStream in, final MessageDigest digest) {
                        if (failed.get()) {
                            // Second attempt successful
                            return in;
                        }
                        return new CountingInputStream(in) {

                            @Override
                            protected void beforeRead(final int n) throws IOException {
                                super.beforeRead(n);
                                if (this.getByteCount() >= 1024L * 1024L) {
                                    failed.set(true);
                                    throw new SocketTimeoutException();
                                }
                            }
                        };
                    }
                };
            }
            return super._getFeature(type);
        }
    };
    session.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback());
    session.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback());
    final Path home = new Path("test-eu-central-1-cyberduck", EnumSet.of(Path.Type.volume, Path.Type.directory));
    final Path test = new Path(home, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final Transfer t = new UploadTransfer(session.getHost(), test, local);
    final BytecountStreamListener counter = new BytecountStreamListener();
    assertTrue(new SingleTransferWorker(session, session, t, new TransferOptions(), new TransferSpeedometer(t), new DisabledTransferPrompt() {

        @Override
        public TransferAction prompt(final TransferItem file) {
            return TransferAction.overwrite;
        }
    }, new DisabledTransferErrorCallback(), new DisabledProgressListener(), counter, new DisabledLoginCallback(), new DisabledNotificationService()) {
    }.run(session));
    local.delete();
    assertTrue(t.isComplete());
    assertEquals(content.length, new S3AttributesFinderFeature(session).find(test).getSize());
    assertEquals(content.length, counter.getRecv(), 0L);
    assertEquals(content.length, counter.getSent(), 0L);
    assertTrue(failed.get());
    new S3DefaultDeleteFeature(session).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

Example 3 with S3MultipartUploadService

use of ch.cyberduck.core.s3.S3MultipartUploadService in project cyberduck by iterate-ch.

the class S3MultipartUploadServiceTest method testUpload.

@Test
public void testUpload() throws Exception {
    // 5L * 1024L * 1024L
    final Path home = new Path("test-eu-central-1-cyberduck", EnumSet.of(Path.Type.volume, Path.Type.directory));
    final Path vault = new Path(home, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory));
    final Path test = new Path(vault, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final CryptoVault cryptomator = new CryptoVault(vault);
    cryptomator.create(session, new VaultCredentials("test"), new DisabledPasswordStore(), vaultVersion);
    session.withRegistry(new DefaultVaultRegistry(new DisabledPasswordStore(), new DisabledPasswordCallback(), cryptomator));
    final CryptoUploadFeature m = new CryptoUploadFeature<>(session, new S3MultipartUploadService(session, new S3WriteFeature(session), 5L * 1024L * 1024L, 5), new S3WriteFeature(session), cryptomator);
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    final byte[] content = RandomUtils.nextBytes(6 * 1024 * 1024);
    IOUtils.write(content, local.getOutputStream(false));
    final TransferStatus writeStatus = new TransferStatus();
    final FileHeader header = cryptomator.getFileHeaderCryptor().create();
    writeStatus.setHeader(cryptomator.getFileHeaderCryptor().encryptHeader(header));
    writeStatus.setLength(content.length);
    m.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), writeStatus, null);
    assertTrue(writeStatus.isComplete());
    assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    final TransferStatus readStatus = new TransferStatus().withLength(content.length);
    final InputStream in = new CryptoReadFeature(session, new S3ReadFeature(session), cryptomator).read(test, readStatus, new DisabledConnectionCallback());
    new StreamCopier(readStatus, readStatus).transfer(in, buffer);
    assertArrayEquals(content, buffer.toByteArray());
    cryptomator.getFeature(session, Delete.class, new S3DefaultDeleteFeature(session)).delete(Arrays.asList(test, vault), new DisabledLoginCallback(), new Delete.DisabledCallback());
    local.delete();
}

Example 4 with S3MultipartUploadService

use of ch.cyberduck.core.s3.S3MultipartUploadService in project cyberduck by iterate-ch.

the class S3MultipartUploadServiceTest method testUploadWithBulk.

@Test
public void testUploadWithBulk() throws Exception {
    // 5L * 1024L * 1024L
    final Path home = new Path("test-eu-central-1-cyberduck", EnumSet.of(Path.Type.volume, Path.Type.directory));
    final Path vault = new Path(home, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory));
    final Path test = new Path(vault, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final CryptoVault cryptomator = new CryptoVault(vault);
    cryptomator.create(session, new VaultCredentials("test"), new DisabledPasswordStore(), vaultVersion);
    session.withRegistry(new DefaultVaultRegistry(new DisabledPasswordStore(), new DisabledPasswordCallback(), cryptomator));
    final TransferStatus writeStatus = new TransferStatus();
    final byte[] content = RandomUtils.nextBytes(6 * 1024 * 1024);
    writeStatus.setLength(content.length);
    final CryptoBulkFeature<Map<TransferItem, TransferStatus>> bulk = new CryptoBulkFeature<>(session, new DisabledBulkFeature(), new S3DefaultDeleteFeature(session), cryptomator);
    bulk.pre(Transfer.Type.upload, Collections.singletonMap(new TransferItem(test), writeStatus), new DisabledConnectionCallback());
    final CryptoUploadFeature m = new CryptoUploadFeature<>(session, new S3MultipartUploadService(session, new S3WriteFeature(session), 5L * 1024L * 1024L, 5), new S3WriteFeature(session), cryptomator);
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    IOUtils.write(content, local.getOutputStream(false));
    m.upload(test, local, new BandwidthThrottle(BandwidthThrottle.UNLIMITED), new DisabledStreamListener(), writeStatus, null);
    assertTrue(writeStatus.isComplete());
    assertTrue(new CryptoFindFeature(session, new S3FindFeature(session), cryptomator).find(test));
    assertEquals(content.length, new CryptoAttributesFeature(session, new S3AttributesFinderFeature(session), cryptomator).find(test).getSize());
    final ByteArrayOutputStream buffer = new ByteArrayOutputStream(content.length);
    final TransferStatus readStatus = new TransferStatus().withLength(content.length);
    final InputStream in = new CryptoReadFeature(session, new S3ReadFeature(session), cryptomator).read(test, readStatus, new DisabledConnectionCallback());
    new StreamCopier(readStatus, readStatus).transfer(in, buffer);
    assertArrayEquals(content, buffer.toByteArray());
    cryptomator.getFeature(session, Delete.class, new S3DefaultDeleteFeature(session)).delete(Arrays.asList(test, vault), new DisabledLoginCallback(), new Delete.DisabledCallback());
    local.delete();
}