Examples with MultichunkRemoteFile org.syncany.plugins.transfer.files.MultichunkRemoteFile used on opensource projects

Example 1 with MultichunkRemoteFile

use of org.syncany.plugins.transfer.files.MultichunkRemoteFile in project syncany by syncany.

the class CleanupOperation method deleteUnusedRemoteMultiChunks.

/**
	 * This method adds unusedMultiChunks to the @{link RemoteTransaction} for deletion.
	 *
	 * @param unusedMultiChunks which are to be deleted because all references to them are gone.
	 */
private void deleteUnusedRemoteMultiChunks(Map<MultiChunkId, MultiChunkEntry> unusedMultiChunks) throws StorageException {
    logger.log(Level.INFO, "- Deleting remote multichunks ...");
    for (MultiChunkEntry multiChunkEntry : unusedMultiChunks.values()) {
        logger.log(Level.FINE, "  + Deleting remote multichunk " + multiChunkEntry + " ...");
        remoteTransaction.delete(new MultichunkRemoteFile(multiChunkEntry.getId()));
    }
}

Example 2 with MultichunkRemoteFile

use of org.syncany.plugins.transfer.files.MultichunkRemoteFile in project syncany by syncany.

the class Downloader method downloadAndDecryptMultiChunks.

/** 
	 * Downloads the given multichunks from the remote storage and decrypts them
	 * to the local cache folder. 
	 */
public void downloadAndDecryptMultiChunks(Set<MultiChunkId> unknownMultiChunkIds) throws StorageException, IOException {
    logger.log(Level.INFO, "Downloading and extracting multichunks ...");
    int multiChunkNumber = 0;
    for (MultiChunkId multiChunkId : unknownMultiChunkIds) {
        File localEncryptedMultiChunkFile = config.getCache().getEncryptedMultiChunkFile(multiChunkId);
        File localDecryptedMultiChunkFile = config.getCache().getDecryptedMultiChunkFile(multiChunkId);
        MultichunkRemoteFile remoteMultiChunkFile = new MultichunkRemoteFile(multiChunkId);
        multiChunkNumber++;
        if (localDecryptedMultiChunkFile.exists()) {
            logger.log(Level.INFO, "  + Decrypted multichunk exists locally " + multiChunkId + ". No need to download it!");
        } else {
            eventBus.post(new DownDownloadFileSyncExternalEvent(config.getLocalDir().getAbsolutePath(), "multichunk", multiChunkNumber, unknownMultiChunkIds.size()));
            logger.log(Level.INFO, "  + Downloading multichunk " + multiChunkId + " ...");
            transferManager.download(remoteMultiChunkFile, localEncryptedMultiChunkFile);
            try {
                logger.log(Level.INFO, "  + Decrypting multichunk " + multiChunkId + " ...");
                InputStream multiChunkInputStream = config.getTransformer().createInputStream(new FileInputStream(localEncryptedMultiChunkFile));
                OutputStream decryptedMultiChunkOutputStream = new FileOutputStream(localDecryptedMultiChunkFile);
                IOUtils.copy(multiChunkInputStream, decryptedMultiChunkOutputStream);
                decryptedMultiChunkOutputStream.close();
                multiChunkInputStream.close();
            } catch (IOException e) {
                // Security: Deleting the multichunk if the decryption/extraction failed is important!
                //           If it is not deleted, the partially decrypted multichunk will reside in the
                //           local cache and the next 'down' will try to use it. If this is the only
                //           multichunk that has been tampered with, other changes might be applied to the 
                //           file system! See https://github.com/syncany/syncany/issues/59#issuecomment-55154793
                logger.log(Level.FINE, "    -> FAILED: Decryption/extraction of multichunk failed, deleting " + multiChunkId + " ...");
                localDecryptedMultiChunkFile.delete();
                throw new IOException("Decryption/extraction of multichunk " + multiChunkId + " failed. The multichunk might have been tampered with!", e);
            } finally {
                logger.log(Level.FINE, "  + Locally deleting multichunk " + multiChunkId + " ...");
                localEncryptedMultiChunkFile.delete();
            }
        }
    }
    transferManager.disconnect();
}

Example 3 with MultichunkRemoteFile

use of org.syncany.plugins.transfer.files.MultichunkRemoteFile in project syncany by syncany.

the class UpOperation method addMultiChunksToTransaction.

/**
	 * This methods adds the multichunks that are not yet present in the remote repo to the {@link RemoteTransaction} for
	 * uploading. Multichunks are not uploaded if they are dirty.
	 *
	 * @param multiChunkEntries Collection of multiChunkEntries that are included in the new {@link DatabaseVersion}
	 */
private void addMultiChunksToTransaction(RemoteTransaction remoteTransaction, Collection<MultiChunkEntry> multiChunksEntries) throws InterruptedException, StorageException {
    List<MultiChunkId> dirtyMultiChunkIds = localDatabase.getDirtyMultiChunkIds();
    for (MultiChunkEntry multiChunkEntry : multiChunksEntries) {
        if (dirtyMultiChunkIds.contains(multiChunkEntry.getId())) {
            logger.log(Level.INFO, "- Ignoring multichunk (from dirty database, already uploaded), " + multiChunkEntry.getId() + " ...");
        } else {
            File localMultiChunkFile = config.getCache().getEncryptedMultiChunkFile(multiChunkEntry.getId());
            MultichunkRemoteFile remoteMultiChunkFile = new MultichunkRemoteFile(multiChunkEntry.getId());
            logger.log(Level.INFO, "- Uploading multichunk {0} from {1} to {2} ...", new Object[] { multiChunkEntry.getId(), localMultiChunkFile, remoteMultiChunkFile });
            remoteTransaction.upload(localMultiChunkFile, remoteMultiChunkFile);
        }
    }
}

Example 4 with MultichunkRemoteFile

use of org.syncany.plugins.transfer.files.MultichunkRemoteFile in project syncany by syncany.

the class UploadInterruptedTest method testUnreliableUpload_Test4_2_FailsAtTXCommitDuring2ndMultiChunkMove.

@Test
public void testUnreliableUpload_Test4_2_FailsAtTXCommitDuring2ndMultiChunkMove() throws Exception {
    /*
		 * First run "Client A": This test fails when trying to execute the TX.commit() when moving the second multichunk. So the first multichunk was
		 * moved successfully.
		 *
		 * Double check by "Client B": Client B should not see this multichunk on TM.list()
		 *
		 * Second run "Client A": The second up() from Client A should revert the transaction. To verify this, we let the second run fail at the
		 * transaction file upload
		 *
		 * 1. upload(action-up-987, actions/action-up-987)
		 * 2. upload(transaction-123, transactions/transaction-123) <<< FAILS HERE (second run)
		 * 3. upload(multichunk-1, temp-1)
		 * 4. upload(multichunk-2, temp-2)
		 * 5. upload(database-123, temp-3)
		 * 6. move(temp-1, multichunks/multichunk-1)
		 * 7. move(temp-2, multichunks/multichunk-2) <<< FAILS HERE (first run)
		 * 8. move(temp-3, databases/database-123)
		 */
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(Arrays.asList(new String[] { "rel=[234].+move.+multichunk", "rel=(7|8|9).+upload.+transaction" }));
    TestClient clientA = new TestClient("A", testConnection);
    // << larger than one multichunk!
    clientA.createNewFile("A-original", 5 * 1024 * 1024);
    boolean firstUpFailed = false;
    try {
        clientA.up();
    } catch (StorageException e) {
        firstUpFailed = true;
        logger.log(Level.INFO, e.getMessage());
    }
    assertTrue(firstUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    File[] tempFiles = new File(testConnection.getPath() + "/temporary/").listFiles();
    assertEquals(2, tempFiles.length);
    // The second multichunk should be >500 KB
    assertTrue(tempFiles[0].length() > 500 * 1024 || tempFiles[1].length() > 500 * 1024);
    // The database file should be <100 KB
    assertTrue(tempFiles[0].length() < 100 * 1024 || tempFiles[1].length() < 100 * 1024);
    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);
    assertEquals(3, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(2).getRemoteFile().getName().contains("database-"));
    // 2. Double check if list() does not return the multichunk
    TransferManager transferManager = TransferManagerFactory.build(clientA.getConfig()).withFeature(TransactionAware.class).asDefault();
    Map<String, MultichunkRemoteFile> multiChunkList = transferManager.list(MultichunkRemoteFile.class);
    assertEquals(0, multiChunkList.size());
    // 3. Second try fails in the beginning, to see if cleanTransactions was successful
    boolean secondUpFailed = false;
    // Do not resume, since we want to clean transactions.
    UpOperationOptions upOptions = new UpOperationOptions();
    upOptions.setResume(false);
    try {
        clientA.up(upOptions);
    } catch (StorageException e) {
        secondUpFailed = true;
        logger.log(Level.INFO, e.getMessage());
    }
    assertTrue(secondUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    // Shouldn't this be 1
    assertEquals(2, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/").listFiles(new FilenameFilter() {

        public boolean accept(File dir, String name) {
            return name.contains("temp-");
        }
    }).length);
    // Tear down
    clientA.deleteTestData();
}

Example 5 with MultichunkRemoteFile

use of org.syncany.plugins.transfer.files.MultichunkRemoteFile in project syncany by syncany.

the class UploadInterruptedTest method testUnreliableUpload_Test4_3_FailsAtTXCommitDuring2ndMultiChunkMoveAndDuringTXRollback.

@Test
public void testUnreliableUpload_Test4_3_FailsAtTXCommitDuring2ndMultiChunkMoveAndDuringTXRollback() throws Exception {
    /*
		 * 1. upload(action-up-987, actions/action-up-987)
		 * 2. upload(transaction-123, transactions/transaction-123)
		 * 3. upload(multichunk-1, temp-1)
		 * 4. upload(multichunk-2, temp-2)
		 * 5. upload(database-123, temp-3)
		 * 6. move(temp-1, multichunks/multichunk-1)
		 * 7. move(temp-2, multichunks/multichunk-2) <<< FAILS HERE (first run)
		 * 8. move(temp-3, databases/database-123)
		 *
		 * 1. upload(action-up-987, actions/action-up-987)
		 * 2. list(databases/*)
		 * 3. list(transactions/*)
		 * 4. upload(transaction-345, transactions/transaction-345) (rollback TX)
		 * 5. move(multichunks/multichunk-1, temp-80)
		 * 6. move(temp-1, temp-81) (silently fails, b/c temp-1 does not exist)
		 * 7. move(multichunks/multichunk-2, temp-82) (silently fails, b/c multichunk-2 does not exist)
		 * 8. move(temp-2, temp-83)
		 * 9. move(databases/database-123, temp-84) (silently fails, b/c database-123 does not exist)
		 * 10. move(temp-3, temp-85)
		 * 10 move(transactions-345, temp-86)
		 * 11. delete(temp-80)
		 * 12. delete(temp-83) <<< FAILS HERE (second run)
		 *
		 * Expected: temp-(83,85,86)
		 */
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(Arrays.asList(new String[] { "rel=[234].+move.+multichunk", "rel=[567].+delete.+database" }));
    TestClient clientA = new TestClient("A", testConnection);
    // << larger than one multichunk!
    clientA.createNewFile("A-original", 5 * 1024 * 1024);
    boolean firstUpFailed = false;
    try {
        clientA.up();
    } catch (StorageException e) {
        firstUpFailed = true;
        logger.log(Level.INFO, e.getMessage());
    }
    assertTrue(firstUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    File[] tempFiles = new File(testConnection.getPath() + "/temporary").listFiles();
    assertEquals(2, tempFiles.length);
    // The second multichunk should be >500 KB
    assertTrue(tempFiles[0].length() > 500 * 1024 || tempFiles[1].length() > 500 * 1024);
    // The database file should be <100 KB
    assertTrue(tempFiles[0].length() < 100 * 1024 || tempFiles[1].length() < 100 * 1024);
    File transactionFile = new File(testConnection.getPath() + "/transactions/").listFiles()[0];
    TransactionTO transactionTO = new Persister().read(TransactionTO.class, transactionFile);
    assertEquals(3, transactionTO.getActions().size());
    assertTrue(transactionTO.getActions().get(0).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(1).getRemoteFile().getName().contains("multichunk-"));
    assertTrue(transactionTO.getActions().get(2).getRemoteFile().getName().contains("database-"));
    // 2. Double check if list() does not return the multichunk
    TransferManager transferManager = TransferManagerFactory.build(clientA.getConfig()).withFeature(TransactionAware.class).asDefault();
    Map<String, MultichunkRemoteFile> multiChunkList = transferManager.list(MultichunkRemoteFile.class);
    assertEquals(0, multiChunkList.size());
    // 3. Second try fails in the beginning, to see if cleanTransactions was successful
    boolean secondUpFailed = false;
    // Do not resume, since we want to clean transactions.
    UpOperationOptions upOptions = new UpOperationOptions();
    upOptions.setResume(false);
    try {
        clientA.up(upOptions);
    } catch (StorageException e) {
        secondUpFailed = true;
        logger.log(Level.INFO, e.getMessage());
    }
    assertTrue(secondUpFailed);
    assertEquals(0, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    // left over, 2 failed ops
    assertEquals(2, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    assertEquals(1, new File(testConnection.getPath() + "/temporary/").listFiles().length);
    // 4. Third try; this should finally succeed
    clientA.up(upOptions);
    assertEquals(1, new File(testConnection.getPath() + "/databases/").listFiles().length);
    assertEquals(2, new File(testConnection.getPath() + "/multichunks/").listFiles().length);
    // cleaned
    assertEquals(0, new File(testConnection.getPath() + "/actions/").listFiles().length);
    assertEquals(0, new File(testConnection.getPath() + "/transactions/").listFiles().length);
    // cleaned
    assertEquals(0, new File(testConnection.getPath() + "/temporary/").listFiles().length);
    // Tear down
    clientA.deleteTestData();
}