Examples with SDSMultipartWriteFeature ch.cyberduck.core.sds.SDSMultipartWriteFeature used on opensource projects

Example 1 with SDSMultipartWriteFeature

use of ch.cyberduck.core.sds.SDSMultipartWriteFeature in project cyberduck by iterate-ch.

the class TripleCryptWriteFeatureTest method testWrite.

@Test
public void testWrite() throws Exception {
    final Path room = new Path("test", EnumSet.of(Path.Type.directory, Path.Type.volume), new PathAttributes().withCustom(KEY_ENCRYPTED, String.valueOf(true)));
    final SDSNodeIdProvider nodeid = new SDSNodeIdProvider(session);
    final TripleCryptWriteFeature writer = new TripleCryptWriteFeature(session, nodeid, new SDSMultipartWriteFeature(session, nodeid));
    final byte[] content = RandomUtils.nextBytes(32769);
    final TransferStatus status = new TransferStatus();
    status.setLength(content.length);
    status.setChecksum(new MD5ChecksumCompute().compute(new ByteArrayInputStream(content), new TransferStatus()));
    final Path test = new Path(room, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final SDSEncryptionBulkFeature bulk = new SDSEncryptionBulkFeature(session, nodeid);
    bulk.pre(Transfer.Type.upload, Collections.singletonMap(new TransferItem(test), status), new DisabledConnectionCallback());
    final StatusOutputStream<Node> out = writer.write(test, status, new DisabledConnectionCallback());
    assertNotNull(out);
    new StreamCopier(status, status).transfer(new ByteArrayInputStream(content), out);
    assertNotNull(test.attributes().getVersionId());
    assertTrue(new DefaultFindFeature(session).find(test));
    assertEquals(content.length, new SDSAttributesFinderFeature(session, nodeid).find(test).getSize());
    final byte[] compare = new byte[content.length];
    final InputStream stream = new TripleCryptReadFeature(session, nodeid, new SDSReadFeature(session, nodeid)).read(test, new TransferStatus(), new DisabledConnectionCallback() {

        @Override
        public void warn(final Host bookmark, final String title, final String message, final String defaultButton, final String cancelButton, final String preference) {
        // 
        }

        @Override
        public Credentials prompt(final Host bookmark, final String title, final String reason, final LoginOptions options) {
            return new VaultCredentials("eth[oh8uv4Eesij");
        }
    });
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new SDSDeleteFeature(session, nodeid).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

Example 2 with SDSMultipartWriteFeature

use of ch.cyberduck.core.sds.SDSMultipartWriteFeature in project cyberduck by iterate-ch.

the class TripleCryptWriteFeatureTest method testWriteMultipart.

@Test
public void testWriteMultipart() throws Exception {
    final Path room = new Path("test", EnumSet.of(Path.Type.directory, Path.Type.volume), new PathAttributes().withCustom(KEY_ENCRYPTED, String.valueOf(true)));
    final byte[] content = RandomUtils.nextBytes(32769);
    final TransferStatus status = new TransferStatus();
    status.setLength(content.length);
    final Path test = new Path(room, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final SDSNodeIdProvider nodeid = new SDSNodeIdProvider(session);
    final SDSEncryptionBulkFeature bulk = new SDSEncryptionBulkFeature(session, nodeid);
    bulk.pre(Transfer.Type.upload, Collections.singletonMap(new TransferItem(test), status), new DisabledConnectionCallback());
    final TripleCryptWriteFeature writer = new TripleCryptWriteFeature(session, nodeid, new SDSMultipartWriteFeature(session, nodeid));
    final StatusOutputStream<Node> out = writer.write(test, status, new DisabledConnectionCallback());
    assertNotNull(out);
    new StreamCopier(status, status).transfer(new ByteArrayInputStream(content), out);
    assertNotNull(test.attributes().getVersionId());
    assertTrue(new DefaultFindFeature(session).find(test));
    assertEquals(content.length, new SDSAttributesFinderFeature(session, nodeid).find(test).getSize());
    final byte[] compare = new byte[content.length];
    final InputStream stream = new TripleCryptReadFeature(session, nodeid, new SDSReadFeature(session, nodeid)).read(test, new TransferStatus(), new DisabledConnectionCallback() {

        @Override
        public void warn(final Host bookmark, final String title, final String message, final String defaultButton, final String cancelButton, final String preference) {
        // 
        }

        @Override
        public Credentials prompt(final Host bookmark, final String title, final String reason, final LoginOptions options) {
            return new VaultCredentials("eth[oh8uv4Eesij");
        }
    });
    IOUtils.readFully(stream, compare);
    stream.close();
    assertArrayEquals(content, compare);
    new SDSDeleteFeature(session, nodeid).delete(Collections.singletonList(test), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

Example 3 with SDSMultipartWriteFeature

use of ch.cyberduck.core.sds.SDSMultipartWriteFeature in project cyberduck by iterate-ch.

the class SDSSingleTransferWorkerTest method testDownloadVersioned.

@Test
public void testDownloadVersioned() throws Exception {
    final SDSNodeIdProvider fileid = new SDSNodeIdProvider(session);
    final Path room = new SDSDirectoryFeature(session, fileid).mkdir(new Path(new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
    final Path test = new Path(room, new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.file));
    final Local localFile = new DefaultTemporaryFileService().create(test.getName());
    {
        final byte[] content = RandomUtils.nextBytes(39864);
        final TransferStatus writeStatus = new TransferStatus().withLength(content.length).withChecksum(new SHA256ChecksumCompute().compute(new ByteArrayInputStream(content), new TransferStatus()));
        final StatusOutputStream<Node> out = new SDSMultipartWriteFeature(session, fileid).write(test, writeStatus, new DisabledConnectionCallback());
        assertNotNull(out);
        new StreamCopier(writeStatus, writeStatus).withLimit((long) content.length).transfer(new ByteArrayInputStream(content), out);
        out.close();
    }
    final byte[] content = RandomUtils.nextBytes(39864);
    final TransferStatus writeStatus = new TransferStatus().exists(true).withLength(content.length).withChecksum(new SHA256ChecksumCompute().compute(new ByteArrayInputStream(content), new TransferStatus()));
    final StatusOutputStream<Node> out = new SDSMultipartWriteFeature(session, fileid).write(test, writeStatus, new DisabledConnectionCallback());
    assertNotNull(out);
    new StreamCopier(writeStatus, writeStatus).withLimit((long) content.length).transfer(new ByteArrayInputStream(content), out);
    out.close();
    final String versionId = test.attributes().getVersionId();
    assertEquals(versionId, new SDSAttributesFinderFeature(session, fileid).find(test).getVersionId());
    assertEquals(versionId, new DefaultAttributesFinderFeature(session).find(test).getVersionId());
    final Transfer t = new DownloadTransfer(new Host(new TestProtocol()), Collections.singletonList(new TransferItem(test, localFile)), new NullFilter<>());
    assertTrue(new SingleTransferWorker(session, session, t, new TransferOptions(), new TransferSpeedometer(t), new DisabledTransferPrompt() {

        @Override
        public TransferAction prompt(final TransferItem file) {
            return TransferAction.overwrite;
        }
    }, new DisabledTransferErrorCallback(), new DisabledProgressListener(), new DisabledStreamListener(), new DisabledLoginCallback(), new DisabledNotificationService()) {
    }.run(session));
    byte[] compare = new byte[content.length];
    assertArrayEquals(content, IOUtils.toByteArray(localFile.getInputStream()));
    test.attributes().setVersionId(versionId);
    assertEquals(versionId, new DefaultAttributesFinderFeature(session).find(test).getVersionId());
    new SDSDeleteFeature(session, fileid).delete(Arrays.asList(test, room), new DisabledLoginCallback(), new Delete.DisabledCallback());
    localFile.delete();
    session.close();
}

Example 4 with SDSMultipartWriteFeature

use of ch.cyberduck.core.sds.SDSMultipartWriteFeature in project cyberduck by iterate-ch.

the class SDSSingleTransferWorkerTest method testTransferredSizeRepeat.

@Test
public void testTransferredSizeRepeat() throws Exception {
    final SDSNodeIdProvider fileid = new SDSNodeIdProvider(session);
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    // chunk size 32768
    final byte[] content = new byte[98305];
    new Random().nextBytes(content);
    final OutputStream out = local.getOutputStream(false);
    IOUtils.write(content, out);
    out.close();
    final AtomicBoolean failed = new AtomicBoolean();
    final BytecountStreamListener counter = new BytecountStreamListener();
    final SDSSession conn = new SDSSession(session.getHost().withCredentials(new Credentials(System.getProperties().getProperty("sds.user"), System.getProperties().getProperty("sds.key"))), new DisabledX509TrustManager(), new DefaultX509KeyManager()) {

        @Override
        @SuppressWarnings("unchecked")
        public <T> T _getFeature(final Class<T> type) {
            if (type == Upload.class) {
                return (T) new DefaultUploadFeature(new SDSMultipartWriteFeature(this, fileid) {

                    @Override
                    public HttpResponseOutputStream<Node> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
                        final HttpResponseOutputStream<Node> proxy = super.write(file, status, callback);
                        if (failed.get()) {
                            // Second attempt successful
                            return proxy;
                        }
                        return new HttpResponseOutputStream<Node>(new CountingOutputStream(proxy) {

                            @Override
                            protected void afterWrite(final int n) throws IOException {
                                super.afterWrite(n);
                                if (this.getByteCount() >= 42768L) {
                                    // Buffer size
                                    assertEquals(32768L, counter.getSent());
                                    failed.set(true);
                                    throw new SocketTimeoutException();
                                }
                            }
                        }, new SDSAttributesAdapter(session), status) {

                            @Override
                            public Node getStatus() throws BackgroundException {
                                return proxy.getStatus();
                            }
                        };
                    }
                });
            }
            return super._getFeature(type);
        }
    };
    conn.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback());
    conn.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback());
    final Path room = new SDSDirectoryFeature(conn, fileid).mkdir(new Path(new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
    final Path test = new Path(room, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final Transfer t = new UploadTransfer(session.getHost(), test, local);
    assertTrue(new SingleTransferWorker(conn, conn, t, new TransferOptions(), new TransferSpeedometer(t), new DisabledTransferPrompt() {

        @Override
        public TransferAction prompt(final TransferItem file) {
            return TransferAction.overwrite;
        }
    }, new DisabledTransferErrorCallback(), new DisabledProgressListener(), counter, new DisabledLoginCallback(), new DisabledNotificationService()) {
    }.run(session));
    local.delete();
    assertTrue(t.isComplete());
    assertEquals(content.length, new SDSAttributesFinderFeature(conn, fileid).find(test).getSize());
    assertEquals(content.length, counter.getRecv(), 0L);
    assertEquals(content.length, counter.getSent(), 0L);
    assertTrue(failed.get());
    new SDSDeleteFeature(conn, fileid).delete(Arrays.asList(test, room), new DisabledLoginCallback(), new Delete.DisabledCallback());
}

Example 5 with SDSMultipartWriteFeature

use of ch.cyberduck.core.sds.SDSMultipartWriteFeature in project cyberduck by iterate-ch.

the class SDSSingleTransferWorkerTest method testTransferredSizeRepeatFailureOnClose.

@Test
public void testTransferredSizeRepeatFailureOnClose() throws Exception {
    final SDSNodeIdProvider fileid = new SDSNodeIdProvider(session);
    final Local local = new Local(System.getProperty("java.io.tmpdir"), UUID.randomUUID().toString());
    // chunk size 32768
    final byte[] content = new byte[98305];
    new Random().nextBytes(content);
    final OutputStream out = local.getOutputStream(false);
    IOUtils.write(content, out);
    out.close();
    final AtomicBoolean failed = new AtomicBoolean();
    final SDSSession conn = new SDSSession(session.getHost().withCredentials(new Credentials(System.getProperties().getProperty("sds.user"), System.getProperties().getProperty("sds.key"))), new DisabledX509TrustManager(), new DefaultX509KeyManager()) {

        @Override
        @SuppressWarnings("unchecked")
        public <T> T _getFeature(final Class<T> type) {
            if (type == Upload.class) {
                return (T) new DefaultUploadFeature<>(new SDSMultipartWriteFeature(this, fileid) {

                    @Override
                    public HttpResponseOutputStream<Node> write(final Path file, final TransferStatus status, final ConnectionCallback callback) throws BackgroundException {
                        final HttpResponseOutputStream<Node> proxy = super.write(file, status, callback);
                        if (failed.get()) {
                            // Second attempt successful
                            return proxy;
                        }
                        return new HttpResponseOutputStream<Node>(new CountingOutputStream(proxy) {

                            @Override
                            public void close() throws IOException {
                                if (!failed.get()) {
                                    failed.set(true);
                                    throw new SocketTimeoutException();
                                }
                                super.close();
                            }
                        }, new SDSAttributesAdapter(session), status) {

                            @Override
                            public Node getStatus() throws BackgroundException {
                                return proxy.getStatus();
                            }
                        };
                    }
                });
            }
            return super._getFeature(type);
        }
    };
    conn.open(Proxy.DIRECT, new DisabledHostKeyCallback(), new DisabledLoginCallback(), new DisabledCancelCallback());
    conn.login(Proxy.DIRECT, new DisabledLoginCallback(), new DisabledCancelCallback());
    final Path room = new SDSDirectoryFeature(conn, fileid).mkdir(new Path(new AlphanumericRandomStringService().random(), EnumSet.of(Path.Type.directory, Path.Type.volume)), new TransferStatus());
    final Path test = new Path(room, UUID.randomUUID().toString(), EnumSet.of(Path.Type.file));
    final Transfer t = new UploadTransfer(session.getHost(), test, local);
    final BytecountStreamListener counter = new BytecountStreamListener();
    assertTrue(new SingleTransferWorker(conn, conn, t, new TransferOptions(), new TransferSpeedometer(t), new DisabledTransferPrompt() {

        @Override
        public TransferAction prompt(final TransferItem file) {
            return TransferAction.overwrite;
        }
    }, new DisabledTransferErrorCallback(), new DisabledProgressListener(), counter, new DisabledLoginCallback(), new DisabledNotificationService()) {
    }.run(conn));
    local.delete();
    assertTrue(t.isComplete());
    assertEquals(98305L, counter.getSent(), 0L);
    assertTrue(failed.get());
    assertEquals(98305L, new SDSAttributesFinderFeature(conn, fileid).find(test).getSize());
    new SDSDeleteFeature(conn, fileid).delete(Arrays.asList(test, room), new DisabledLoginCallback(), new Delete.DisabledCallback());
}