Examples with CleanupOperationOptions org.syncany.operations.cleanup.CleanupOperationOptions used on opensource projects

Example 11 with CleanupOperationOptions

use of org.syncany.operations.cleanup.CleanupOperationOptions in project syncany by syncany.

the class CleanupInterruptedTest method testUnreliableCleanup_Test2_databaseFileMerge.

@Test
public void testUnreliableCleanup_Test2_databaseFileMerge() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(Arrays.asList(new String[] { // << 3 retries!!
    "rel=(11|12|13).+upload.+database" }));
    TestClient clientA = new TestClient("A", testConnection);
    clientA.createNewFile("file");
    clientA.up();
    clientA.changeFile("file");
    clientA.upWithForceChecksum();
    CleanupOperationOptions cleanupOptions = new CleanupOperationOptions();
    cleanupOptions.setMaxDatabaseFiles(1);
    cleanupOptions.setPurgeFileVersionSettings(new TreeMap<Long, TimeUnit>());
    boolean cleanupFailed = false;
    try {
        clientA.cleanup(cleanupOptions);
    } catch (StorageException e) {
        cleanupFailed = true;
    }
    assertTrue(cleanupFailed);
    TransferManager transferManagerA = TransferManagerFactory.build(clientA.getConfig()).withFeature(TransactionAware.class).asDefault();
    assertEquals(2, transferManagerA.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);
    // Note that the list here differs from the actual files, because the transaction fails
    // while deletions have been done
    assertEquals(2, transferManagerA.list(DatabaseRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "databases").list().length);
    assertEquals(1, transferManagerA.list(TransactionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "transactions").list().length);
    assertEquals(1, transferManagerA.list(ActionRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "actions").list().length);
    File tempDir = TestFileUtil.createTempDirectoryInSystemTemp();
    File tempFile = File.createTempFile("multichunk", "", tempDir);
    for (RemoteFile remoteFile : transferManagerA.list(DatabaseRemoteFile.class).values()) {
        transferManagerA.download(remoteFile, tempFile);
        assertTrue(tempFile.exists());
        tempFile.delete();
    }
    // Cleanup should have merged the two files.
    CleanupOperationResult result = clientA.cleanup(cleanupOptions);
    assertEquals(CleanupResultCode.OK, result.getResultCode());
    assertEquals(2, result.getMergedDatabaseFilesCount());
    assertEquals(2, transferManagerA.list(MultichunkRemoteFile.class).size());
    assertEquals(2, new File(testConnection.getPath(), "multichunks").list().length);
    assertEquals(1, transferManagerA.list(DatabaseRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "databases").list(new FilenameFilter() {

        public boolean accept(File dir, String name) {
            return name.startsWith("database-");
        }
    }).length);
    assertEquals(0, transferManagerA.list(TransactionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "transactions").list().length);
    assertEquals(0, transferManagerA.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);
    clientA.deleteTestData();
}

Example 12 with CleanupOperationOptions

use of org.syncany.operations.cleanup.CleanupOperationOptions in project syncany by syncany.

the class CleanupInterruptedTest method testUnreliableCleanup_Test3_unreferencedTempFiles.

@Test
public void testUnreliableCleanup_Test3_unreferencedTempFiles() throws Exception {
    // Setup
    UnreliableLocalTransferSettings testConnection = TestConfigUtil.createTestUnreliableLocalConnection(Arrays.asList(new String[] { // << 3 retries!!
    "rel=[678].+delete.+temp" }));
    TestClient clientA = new TestClient("A", testConnection);
    clientA.createNewFile("file");
    clientA.up();
    clientA.changeFile("file");
    clientA.upWithForceChecksum();
    CleanupOperationOptions cleanupOptions = new CleanupOperationOptions();
    cleanupOptions.setMinSecondsBetweenCleanups(0);
    cleanupOptions.setMinKeepSeconds(0);
    clientA.cleanup(cleanupOptions);
    TransferManager transferManagerA = TransferManagerFactory.build(clientA.getConfig()).withFeature(TransactionAware.class).asDefault();
    assertEquals(1, transferManagerA.list(MultichunkRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "multichunks").list().length);
    assertEquals(1, transferManagerA.list(DatabaseRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "databases").list(new FilenameFilter() {

        public boolean accept(File dir, String name) {
            return name.startsWith("database-");
        }
    }).length);
    assertEquals(0, transferManagerA.list(TransactionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "transactions").list().length);
    assertEquals(0, transferManagerA.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);
    // One deletion failed
    assertEquals(1, transferManagerA.list(TempRemoteFile.class).size());
    assertEquals(1, new File(testConnection.getPath(), "temporary").list().length);
    // Change something to trigger cleanup
    clientA.changeFile("file");
    clientA.up();
    CleanupOperationResult result = clientA.cleanup(cleanupOptions);
    // Functional cleanup results in removal of action file and unreferenced files
    assertEquals(0, transferManagerA.list(ActionRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "actions").list().length);
    assertEquals(0, transferManagerA.list(TempRemoteFile.class).size());
    assertEquals(0, new File(testConnection.getPath(), "temporary").list().length);
    clientA.deleteTestData();
}

Example 13 with CleanupOperationOptions

use of org.syncany.operations.cleanup.CleanupOperationOptions in project syncany by syncany.

the class CleanupMergeDatabaseFilesScenarioTest method testIssue58_2.

@Test
public void testIssue58_2() throws Exception {
    /*
		 * This is the attempt to reproduce issue #58 https://github.com/syncany/syncany/issues/58
		 */
    // Setup
    LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection();
    TestClient clientA = new TestClient("A", testConnection);
    TestClient clientB = new TestClient("B", testConnection);
    TestClient clientC = new TestClient("C", testConnection);
    java.sql.Connection databaseConnectionA = DatabaseConnectionFactory.createConnection(clientA.getDatabaseFile(), false);
    java.sql.Connection databaseConnectionB = DatabaseConnectionFactory.createConnection(clientB.getDatabaseFile(), false);
    CleanupOperationOptions options = new CleanupOperationOptions();
    options.setRemoveOldVersions(true);
    options.setMinSecondsBetweenCleanups(0);
    options.setMaxDatabaseFiles(7);
    StatusOperationOptions statusOptionsForceChecksum = new StatusOperationOptions();
    statusOptionsForceChecksum.setForceChecksum(true);
    UpOperationOptions upOperationOptionsWithCleanupForce = new UpOperationOptions();
    upOperationOptionsWithCleanupForce.setStatusOptions(statusOptionsForceChecksum);
    upOperationOptionsWithCleanupForce.setForceUploadEnabled(true);
    // Run preparations
    clientA.down();
    clientA.createNewFile("A-file.jpg");
    // (A1)
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A2)
    clientA.up(upOperationOptionsWithCleanupForce);
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A2,B1)
    clientB.up(upOperationOptionsWithCleanupForce);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A3,B1)
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A4,B1)
    clientA.up(upOperationOptionsWithCleanupForce);
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A4,B2) + (A4,B3) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    clientC.down();
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A5,B3) + (A6,B3) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A6,B4) + (A6,B5) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    /*
		 * For some reason, this chunk checksum in the following commit is the reason for the exception. So we record it here to see where it vanishes
		 * from the databases.
		 */
    clientA.down();
    clientA.changeFile("A-file.jpg");
    String fileAndChunkChecksumThatRaisesException = StringUtil.toHex(TestFileUtil.createChecksum(clientA.getLocalFile("A-file.jpg")));
    System.out.println("Chunk/File checksum that raises the issue: " + fileAndChunkChecksumThatRaisesException);
    clientA.createNewFile("ADDED_IN_DBV_A7_B5");
    // (A7,B5) + (A8,B5) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    assertEquals("1", TestSqlUtil.runSqlSelect("select count(*) from chunk where checksum='" + fileAndChunkChecksumThatRaisesException + "'", databaseConnectionA));
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A8,B6) + (A8,B7) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    assertEquals("0", TestSqlUtil.runSqlSelect("select count(*) from chunk where checksum='" + fileAndChunkChecksumThatRaisesException + "'", databaseConnectionB));
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A9,B7) + (A10,B7) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    assertEquals("0", TestSqlUtil.runSqlSelect("select count(*) from chunk where checksum='" + fileAndChunkChecksumThatRaisesException + "'", databaseConnectionA));
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A10,B8) + (A10,B9) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    assertEquals("0", TestSqlUtil.runSqlSelect("select count(*) from chunk where checksum='" + fileAndChunkChecksumThatRaisesException + "'", databaseConnectionB));
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A10,B10) + (A10,B11) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    assertEquals("0", TestSqlUtil.runSqlSelect("select count(*) from chunk where checksum='" + fileAndChunkChecksumThatRaisesException + "'", databaseConnectionB));
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A11,B11) + (A12,B11) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A13,B11) + (A14,B11) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    clientB.down();
    clientB.changeFile("A-file.jpg");
    // (A14,B12) + (A14,B13) [PURGE]
    clientB.up(upOperationOptionsWithCleanupForce);
    clientB.cleanup(options);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A15,B13) + (A16,B13) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    clientA.down();
    clientA.changeFile("A-file.jpg");
    // (A17,B13) + (A18,B13) [PURGE]
    clientA.up(upOperationOptionsWithCleanupForce);
    clientA.cleanup(options);
    // Sync them up
    clientA.down();
    clientB.down();
    assertSqlDatabaseEquals(clientA.getDatabaseFile(), clientB.getDatabaseFile());
    // Run
    // <<< Here is/was the issue: Client C failed when downloading
    clientC.down();
    assertSqlDatabaseEquals(clientA.getDatabaseFile(), clientC.getDatabaseFile());
    // Tear down
    clientA.deleteTestData();
    clientB.deleteTestData();
    clientC.deleteTestData();
}

Example 14 with CleanupOperationOptions

use of org.syncany.operations.cleanup.CleanupOperationOptions in project syncany by syncany.

the class CleanupMergeDatabaseFilesScenarioTest method testIssue266_EmptyDatabaseAfterCleanup.

@Test
public void testIssue266_EmptyDatabaseAfterCleanup() throws Exception {
    // Test for https://github.com/syncany/syncany/issues/266#issuecomment-64472059
    // Setup
    LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection();
    TestClient clientA = new TestClient("A", testConnection);
    TestClient clientB = new TestClient("B", testConnection);
    TestClient clientC = new TestClient("C", testConnection);
    TestClient clientD = new TestClient("D", testConnection);
    TestClient clientE = new TestClient("E", testConnection);
    CleanupOperationOptions cleanupOptionsKeepOneForce = new CleanupOperationOptions();
    cleanupOptionsKeepOneForce.setRemoveOldVersions(true);
    cleanupOptionsKeepOneForce.setForce(true);
    cleanupOptionsKeepOneForce.setMinKeepSeconds(0);
    // Create a couple of files, then delete them and do a cleanup
    clientA.createNewFile("fileA");
    clientA.upWithForceChecksum();
    clientB.down();
    clientB.createNewFile("fileB");
    clientB.upWithForceChecksum();
    clientC.down();
    clientC.createNewFile("fileC");
    clientC.upWithForceChecksum();
    clientD.down();
    clientD.deleteFile("fileA");
    clientD.deleteFile("fileB");
    clientD.deleteFile("fileC");
    clientD.upWithForceChecksum();
    clientD.cleanup(cleanupOptionsKeepOneForce);
    java.sql.Connection databaseConnectionD = DatabaseConnectionFactory.createConnection(clientD.getDatabaseFile(), false);
    assertEquals("A,2\nB,2\nC,2\nD,2", TestSqlUtil.runSqlSelect("select client, filenumber from known_databases order by client, filenumber", databaseConnectionD));
    assertEquals("", TestSqlUtil.runSqlSelect("select vectorclock_serialized from databaseversion", databaseConnectionD));
    // Now the remote databases are completely empty (no files, no histories, no database versions!)
    /* Case 1: 
		 * 
		 * Client A now knows "fileA" and must react on the cleanup by client D.
		 * The remote databases do NOT contain any trace of "fileA" anymore, so
		 * client A has to detect the deletion by comparing the local database with 
		 * the winner database. "fileA" should be deleted after the next 'down'.
		 */
    // Existing client  << This created a NullPointerException in #266
    clientA.down();
    assertFalse("File 'fileA' should have been deleted.", clientA.getLocalFile("fileA").exists());
    assertFalse("File 'fileB' should not have been created.", clientA.getLocalFile("fileB").exists());
    assertFalse("File 'fileC' should not have been created.", clientA.getLocalFile("fileC").exists());
    assertConflictingFileNotExists("fileA", clientA.getLocalFiles());
    assertConflictingFileNotExists("fileB", clientA.getLocalFiles());
    assertConflictingFileNotExists("fileC", clientA.getLocalFiles());
    assertSqlDatabaseEquals(clientD.getDatabaseFile(), clientA.getDatabaseFile());
    assertFileListEquals(clientD.getLocalFiles(), clientA.getLocalFiles());
    java.sql.Connection databaseConnectionA = DatabaseConnectionFactory.createConnection(clientA.getDatabaseFile(), false);
    assertEquals("A,2\nB,2\nC,2\nD,2", TestSqlUtil.runSqlSelect("select client, filenumber from known_databases order by client, filenumber", databaseConnectionA));
    /*
		 * Case 2:
		 * 
		 * Client E is a completely new client. It's the first time downloading anything, so
		 * it has no local database, and (in this case), the remote/winner database is completely
		 * empty!
		 */
    // Empty/new client << This created a NullPointerException 
    clientE.down();
    assertFalse("File 'fileA' should not have been created.", clientE.getLocalFile("fileA").exists());
    assertFalse("File 'fileB' should not have been created.", clientE.getLocalFile("fileB").exists());
    assertFalse("File 'fileC' should not have been created.", clientE.getLocalFile("fileC").exists());
    assertConflictingFileNotExists("fileA", clientA.getLocalFiles());
    assertConflictingFileNotExists("fileB", clientA.getLocalFiles());
    assertConflictingFileNotExists("fileC", clientA.getLocalFiles());
    assertSqlDatabaseEquals(clientD.getDatabaseFile(), clientE.getDatabaseFile());
    assertFileListEquals(clientD.getLocalFiles(), clientE.getLocalFiles());
    java.sql.Connection databaseConnectionE = DatabaseConnectionFactory.createConnection(clientE.getDatabaseFile(), false);
    assertEquals("A,2\nB,2\nC,2\nD,2", TestSqlUtil.runSqlSelect("select client, filenumber from known_databases order by client, filenumber", databaseConnectionE));
    // After a successful down, create a new database version (continue numbering!)
    clientA.createNewFile("fileA");
    UpOperationResult upResult = clientA.upWithForceChecksum();
    assertEquals(UpResultCode.OK_CHANGES_UPLOADED, upResult.getResultCode());
    assertEquals("(A3,B2,C2,D2)", TestSqlUtil.runSqlSelect("select vectorclock_serialized from databaseversion", databaseConnectionA));
    // Check if E applies everything correctly and check E's numbering
    clientE.down();
    assertSqlDatabaseEquals(clientA.getDatabaseFile(), clientE.getDatabaseFile());
    assertFileListEquals(clientA.getLocalFiles(), clientE.getLocalFiles());
    assertEquals("A,2\nA,3\nB,2\nC,2\nD,2", TestSqlUtil.runSqlSelect("select client, filenumber from known_databases order by client, filenumber", databaseConnectionE));
    clientE.changeFile("fileA");
    upResult = clientE.upWithForceChecksum();
    assertEquals(UpResultCode.OK_CHANGES_UPLOADED, upResult.getResultCode());
    assertEquals("(A3,B2,C2,D2)\n(A3,B2,C2,D2,E1)", TestSqlUtil.runSqlSelect("select vectorclock_serialized from databaseversion", databaseConnectionE));
    // And with D ...
    clientD.down();
    assertSqlDatabaseEquals(clientE.getDatabaseFile(), clientD.getDatabaseFile());
    assertFileListEquals(clientE.getLocalFiles(), clientD.getLocalFiles());
    assertEquals("A,2\nA,3\nB,2\nC,2\nD,2\nE,1", TestSqlUtil.runSqlSelect("select client, filenumber from known_databases order by client, filenumber", databaseConnectionD));
    assertEquals("(A3,B2,C2,D2)\n(A3,B2,C2,D2,E1)", TestSqlUtil.runSqlSelect("select vectorclock_serialized from databaseversion", databaseConnectionD));
    // Tear down
    clientA.deleteTestData();
    clientB.deleteTestData();
    clientC.deleteTestData();
    clientD.deleteTestData();
    clientE.deleteTestData();
}

Example 15 with CleanupOperationOptions

use of org.syncany.operations.cleanup.CleanupOperationOptions in project syncany by syncany.

the class CleanupMergeDatabaseFilesScenarioTest method testDeleteFileAndCleanup.

@Test
public void testDeleteFileAndCleanup() throws Exception {
    // Test if a deleted file is deleted remotely even after a cleanup
    // Setup
    LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection();
    TestClient clientA = new TestClient("A", testConnection);
    TestClient clientB = new TestClient("B", testConnection);
    CleanupOperationOptions cleanupOptionsKeepOneForce = new CleanupOperationOptions();
    cleanupOptionsKeepOneForce.setRemoveOldVersions(true);
    cleanupOptionsKeepOneForce.setForce(true);
    // Create a couple of files, then delete them and do a cleanup
    clientA.createNewFile("fileA1");
    clientA.createNewFile("fileA2");
    clientA.upWithForceChecksum();
    clientB.down();
    clientB.deleteFile("fileA1");
    clientB.upWithForceChecksum();
    // <<< This accidentally(?) deletes file histories marked DELETED
    clientB.cleanup(cleanupOptionsKeepOneForce);
    clientA.down();
    assertFalse("Deleted file still exists.", clientA.getLocalFile("fileA1").exists());
    // Tear down
    clientA.deleteTestData();
    clientB.deleteTestData();
}