Examples with MultiChunk org.syncany.chunk.MultiChunk used on opensource projects

Example 1 with MultiChunk

use of org.syncany.chunk.MultiChunk in project syncany by syncany.

the class Assembler method assembleToCache.

/**
	 * Assembles the given file version to the local cache and returns a reference
	 * to the cached file after successfully assembling the file. 
	 */
public File assembleToCache(FileVersion fileVersion) throws Exception {
    File reconstructedFileInCache = config.getCache().createTempFile("reconstructedFileVersion");
    logger.log(Level.INFO, "     - Creating file " + fileVersion.getPath() + " to " + reconstructedFileInCache + " ...");
    FileContent fileContent = localDatabase.getFileContent(fileVersion.getChecksum(), true);
    if (fileContent == null && memoryDatabase != null) {
        fileContent = memoryDatabase.getContent(fileVersion.getChecksum());
    }
    // Check consistency!
    if (fileContent == null && fileVersion.getChecksum() != null) {
        throw new Exception("Cannot determine file content for checksum " + fileVersion.getChecksum());
    }
    // Create empty file
    if (fileContent == null) {
        FileUtils.touch(reconstructedFileInCache);
        return reconstructedFileInCache;
    }
    // Create non-empty file
    Chunker chunker = config.getChunker();
    MultiChunker multiChunker = config.getMultiChunker();
    FileOutputStream reconstructedFileOutputStream = new FileOutputStream(reconstructedFileInCache);
    MessageDigest reconstructedFileChecksum = MessageDigest.getInstance(chunker.getChecksumAlgorithm());
    if (fileContent != null) {
        // File can be empty!
        Collection<ChunkChecksum> fileChunks = fileContent.getChunks();
        for (ChunkChecksum chunkChecksum : fileChunks) {
            MultiChunkId multiChunkIdForChunk = localDatabase.getMultiChunkId(chunkChecksum);
            if (multiChunkIdForChunk == null && memoryDatabase != null) {
                multiChunkIdForChunk = memoryDatabase.getMultiChunkIdForChunk(chunkChecksum);
            }
            File decryptedMultiChunkFile = config.getCache().getDecryptedMultiChunkFile(multiChunkIdForChunk);
            MultiChunk multiChunk = multiChunker.createMultiChunk(decryptedMultiChunkFile);
            InputStream chunkInputStream = multiChunk.getChunkInputStream(chunkChecksum.getBytes());
            byte[] buffer = new byte[4096];
            int read = 0;
            while (-1 != (read = chunkInputStream.read(buffer))) {
                reconstructedFileChecksum.update(buffer, 0, read);
                reconstructedFileOutputStream.write(buffer, 0, read);
            }
            chunkInputStream.close();
            multiChunk.close();
        }
    }
    reconstructedFileOutputStream.close();
    // Validate checksum
    byte[] reconstructedFileExpectedChecksum = fileContent.getChecksum().getBytes();
    byte[] reconstructedFileActualChecksum = reconstructedFileChecksum.digest();
    if (!Arrays.equals(reconstructedFileActualChecksum, reconstructedFileExpectedChecksum)) {
        throw new Exception("Checksums do not match: expected " + StringUtil.toHex(reconstructedFileExpectedChecksum) + " != actual " + StringUtil.toHex(reconstructedFileActualChecksum));
    }
    return reconstructedFileInCache;
}

Example 2 with MultiChunk

use of org.syncany.chunk.MultiChunk in project syncany by syncany.

the class AssemblerTest method testAssembler.

/**
	 * Alters multichunk data to test whether integrity checks work.
	 */
@Test
public void testAssembler() throws Exception {
    LocalTransferSettings testConnection = (LocalTransferSettings) TestConfigUtil.createTestLocalConnection();
    TestClient clientA = new TestClient("A", testConnection);
    TestClient clientB = new TestClient("B", testConnection);
    // small, only one chunk, one multichunk
    clientA.createNewFile("file1.jpg", 20);
    clientA.upWithForceChecksum();
    // Read chunk from original multichunk
    File repoMultiChunksFolder = new File(testConnection.getPath(), "multichunks");
    File multiChunkFile = repoMultiChunksFolder.listFiles()[0];
    MultiChunk multiChunk = clientA.getConfig().getMultiChunker().createMultiChunk(clientA.getConfig().getTransformer().createInputStream(new FileInputStream(multiChunkFile)));
    Chunk chunk = multiChunk.read();
    multiChunk.close();
    // Flip byte in chunk and write new "altered" multichunk
    File alteredMultiChunkFile = new File(multiChunkFile + "-altered");
    MultiChunk alteredMultiChunk = clientA.getConfig().getMultiChunker().createMultiChunk(multiChunk.getId(), clientA.getConfig().getTransformer().createOutputStream(new FileOutputStream(alteredMultiChunkFile)));
    // Flip one byte!
    chunk.getContent()[0] ^= 0x01;
    alteredMultiChunk.write(chunk);
    alteredMultiChunk.close();
    // Now delete old multichunk, and swap by "altered" file
    multiChunkFile.delete();
    FileUtils.moveFile(alteredMultiChunkFile, multiChunkFile);
    boolean exceptionThrown = false;
    try {
        // If this does not throw an exception, it's bad!
        clientB.down();
    } catch (Exception e) {
        exceptionThrown = true;
    }
    assertTrue(exceptionThrown);
    clientA.deleteTestData();
    clientB.deleteTestData();
}

Example 3 with MultiChunk

use of org.syncany.chunk.MultiChunk in project syncany by syncany.

the class FrameworkCombinationTest method deduplicateAndCreateChunkIndex.

private ChunkIndex deduplicateAndCreateChunkIndex(final List<File> inputFiles, FrameworkCombination combination) throws IOException {
    logger.log(Level.INFO, "- Deduplicate and create chunk index ...");
    final ChunkIndex chunkIndex = new ChunkIndex();
    Deduper deduper = new Deduper(combination.chunker, combination.multiChunker, combination.transformer, Long.MAX_VALUE, Long.MAX_VALUE);
    deduper.deduplicate(inputFiles, new DeduperListener() {

        @Override
        public void onMultiChunkWrite(MultiChunk multiChunk, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to multichunk " + multiChunk.getId() + " ...");
            chunkIndex.chunkIDToMultiChunkID.put(new ChunkChecksum(chunk.getChecksum()), multiChunk.getId());
        }

        @Override
        public void onFileAddChunk(File file, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to inputFileToChunkIDs-map for file " + file + " ...");
            List<ChunkChecksum> chunkIDsForFile = chunkIndex.inputFileToChunkIDs.get(file);
            if (chunkIDsForFile == null) {
                chunkIDsForFile = new ArrayList<ChunkChecksum>();
            }
            chunkIDsForFile.add(new ChunkChecksum(chunk.getChecksum()));
            chunkIndex.inputFileToChunkIDs.put(file, chunkIDsForFile);
        }

        @Override
        public boolean onChunk(Chunk chunk) {
            if (chunkIndex.chunkIDToMultiChunkID.containsKey(new ChunkChecksum(chunk.getChecksum()))) {
                logger.log(Level.INFO, "  + Known chunk " + StringUtil.toHex(chunk.getChecksum()));
                return false;
            } else {
                logger.log(Level.INFO, "  + New chunk " + StringUtil.toHex(chunk.getChecksum()));
                return true;
            }
        }

        @Override
        public File getMultiChunkFile(MultiChunkId multiChunkId) {
            File outputMultiChunk = new File(tempDir + "/multichunk-" + multiChunkId);
            chunkIndex.outputMultiChunkFiles.add(outputMultiChunk);
            return outputMultiChunk;
        }

        @Override
        public MultiChunkId createNewMultiChunkId(Chunk firstChunk) {
            // Note: In the real implementation, this should be random
            return new MultiChunkId(firstChunk.getChecksum());
        }

        @Override
        public boolean onFileFilter(File file) {
            return true;
        }

        @Override
        public boolean onFileStart(File file) {
            return file.isFile() && !FileUtil.isSymlink(file);
        }

        @Override
        public void onFileEnd(File file, byte[] checksum) {
        // Empty
        }

        @Override
        public void onMultiChunkOpen(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onMultiChunkClose(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onStart(int fileCount) {
        // Empty
        }

        @Override
        public void onFinish() {
        // Empty
        }
    });
    return chunkIndex;
}

Example 4 with MultiChunk

use of org.syncany.chunk.MultiChunk in project syncany by syncany.

the class MultiChunkerTest method chunkFileIntoMultiChunks.

private Set<MultiChunk> chunkFileIntoMultiChunks(File tempDir, List<File> files, Chunker foc, MultiChunker customMultiChunker, Transformer transformer) throws IOException {
    Set<MultiChunk> resultMultiChunks = new HashSet<MultiChunk>();
    MultiChunk customMultiChunk = createNewMultiChunk(tempDir, customMultiChunker, transformer);
    for (File file : files) {
        Enumeration<Chunk> chunks = foc.createChunks(file);
        while (chunks.hasMoreElements()) {
            Chunk chunk = chunks.nextElement();
            customMultiChunk.write(chunk);
            if (customMultiChunk.isFull()) {
                customMultiChunk.close();
                resultMultiChunks.add(customMultiChunk);
                customMultiChunk = createNewMultiChunk(tempDir, customMultiChunker, transformer);
            }
        }
    }
    customMultiChunk.close();
    resultMultiChunks.add(customMultiChunk);
    return resultMultiChunks;
}

Example 5 with MultiChunk

use of org.syncany.chunk.MultiChunk in project syncany by syncany.

the class MultiChunkerTest method createNewMultiChunk.

private MultiChunk createNewMultiChunk(File tempDir, MultiChunker customMultiChunker, Transformer transformer) {
    FileOutputStream fos;
    MultiChunkId multiChunkId = MultiChunkId.secureRandomMultiChunkId();
    MultiChunk customChunk = null;
    try {
        fos = new FileOutputStream(tempDir.getAbsolutePath() + "/MultiChunk" + multiChunkId);
        customChunk = customMultiChunker.createMultiChunk(multiChunkId, transformer.createOutputStream(fos));
    } catch (FileNotFoundException e) {
        e.printStackTrace();
    } catch (IOException e) {
        e.printStackTrace();
    }
    assertEquals(customChunk.getId(), multiChunkId);
    return customChunk;
}