Examples with MultiChunker org.syncany.chunk.MultiChunker used on opensource projects

Example 1 with MultiChunker

use of org.syncany.chunk.MultiChunker in project syncany by syncany.

the class FrameworkCombinationTest method fillCombinations.

private void fillCombinations() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException {
    // MultiChunks
    List<MultiChunker> multiChunkers = new LinkedList<MultiChunker>();
    int[] multiChunkSizes = new int[] { 250000, 500000 };
    for (int i = 0; i < multiChunkSizes.length; i++) {
        // multiChunkers.add(new CustomMultiChunker(multiChunkSizes[i]));
        multiChunkers.add(new ZipMultiChunker(multiChunkSizes[i]));
    }
    // Chunks
    List<Chunker> chunkers = new LinkedList<Chunker>();
    int[] chunkSizes = new int[] { 8000, 16000 };
    String[] digestAlgs = new String[] { /*"MD5" ,*/
    "SHA1" };
    String[] fingerprinters = new String[] { "Adler32" /*, "Plain", "Rabin"*/
    };
    for (int i = 0; i < chunkSizes.length; i++) {
        for (int j = 0; j < digestAlgs.length; j++) {
            for (int k = 0; k < fingerprinters.length; k++) {
                chunkers.add(new TttdChunker(chunkSizes[i], TttdChunker.DEFAULT_WINDOW_SIZE, digestAlgs[j], fingerprinters[k]));
            }
        }
    }
    // Compression/Encryption
    List<CipherSpec> cipherSpecs = new ArrayList<CipherSpec>();
    cipherSpecs.add(CipherSpecs.getCipherSpec(1));
    cipherSpecs.add(CipherSpecs.getCipherSpec(2));
    List<Transformer> transformerChains = new LinkedList<Transformer>();
    transformerChains.add(new NoTransformer());
    transformerChains.add(new GzipTransformer());
    transformerChains.add(new CipherTransformer(cipherSpecs, masterKey));
    transformerChains.add(new GzipTransformer(new CipherTransformer(cipherSpecs, masterKey)));
    for (MultiChunker multiChunker : multiChunkers) {
        for (Transformer transformer : transformerChains) {
            for (Chunker chunker : chunkers) {
                String configName = multiChunker + "/" + chunker + "/" + transformer;
                combinations.add(new FrameworkCombination(configName, chunker, multiChunker, transformer));
            }
        }
    }
}

Example 2 with MultiChunker

use of org.syncany.chunk.MultiChunker in project syncany by syncany.

the class Assembler method assembleToCache.

/**
 * Assembles the given file version to the local cache and returns a reference
 * to the cached file after successfully assembling the file.
 */
public File assembleToCache(FileVersion fileVersion) throws Exception {
    File reconstructedFileInCache = config.getCache().createTempFile("reconstructedFileVersion");
    logger.log(Level.INFO, "     - Creating file " + fileVersion.getPath() + " to " + reconstructedFileInCache + " ...");
    FileContent fileContent = localDatabase.getFileContent(fileVersion.getChecksum(), true);
    if (fileContent == null && memoryDatabase != null) {
        fileContent = memoryDatabase.getContent(fileVersion.getChecksum());
    }
    // Check consistency!
    if (fileContent == null && fileVersion.getChecksum() != null) {
        throw new Exception("Cannot determine file content for checksum " + fileVersion.getChecksum());
    }
    // Create empty file
    if (fileContent == null) {
        FileUtils.touch(reconstructedFileInCache);
        return reconstructedFileInCache;
    }
    // Create non-empty file
    Chunker chunker = config.getChunker();
    MultiChunker multiChunker = config.getMultiChunker();
    FileOutputStream reconstructedFileOutputStream = new FileOutputStream(reconstructedFileInCache);
    MessageDigest reconstructedFileChecksum = MessageDigest.getInstance(chunker.getChecksumAlgorithm());
    if (fileContent != null) {
        // File can be empty!
        Collection<ChunkChecksum> fileChunks = fileContent.getChunks();
        for (ChunkChecksum chunkChecksum : fileChunks) {
            MultiChunkId multiChunkIdForChunk = localDatabase.getMultiChunkId(chunkChecksum);
            if (multiChunkIdForChunk == null && memoryDatabase != null) {
                multiChunkIdForChunk = memoryDatabase.getMultiChunkIdForChunk(chunkChecksum);
            }
            File decryptedMultiChunkFile = config.getCache().getDecryptedMultiChunkFile(multiChunkIdForChunk);
            MultiChunk multiChunk = multiChunker.createMultiChunk(decryptedMultiChunkFile);
            InputStream chunkInputStream = multiChunk.getChunkInputStream(chunkChecksum.getBytes());
            byte[] buffer = new byte[4096];
            int read = 0;
            while (-1 != (read = chunkInputStream.read(buffer))) {
                reconstructedFileChecksum.update(buffer, 0, read);
                reconstructedFileOutputStream.write(buffer, 0, read);
            }
            chunkInputStream.close();
            multiChunk.close();
        }
    }
    reconstructedFileOutputStream.close();
    // Validate checksum
    byte[] reconstructedFileExpectedChecksum = fileContent.getChecksum().getBytes();
    byte[] reconstructedFileActualChecksum = reconstructedFileChecksum.digest();
    if (!Arrays.equals(reconstructedFileActualChecksum, reconstructedFileExpectedChecksum)) {
        throw new Exception("Checksums do not match: expected " + StringUtil.toHex(reconstructedFileExpectedChecksum) + " != actual " + StringUtil.toHex(reconstructedFileActualChecksum));
    }
    return reconstructedFileInCache;
}

Example 3 with MultiChunker

use of org.syncany.chunk.MultiChunker in project syncany by syncany.

the class MultiChunkerTest method testChunkFileIntoMultiChunks.

@Test
public void testChunkFileIntoMultiChunks() throws Exception {
    int minMultiChunkSize = 512;
    int chunkSizeB = 16000;
    Chunker[] chunkers = new Chunker[] { new FixedChunker(chunkSizeB) };
    MultiChunker[] multiChunkers = new MultiChunker[] { // new CustomMultiChunker(minMultiChunkSize),
    new ZipMultiChunker(minMultiChunkSize) };
    for (Chunker chunker : chunkers) {
        for (MultiChunker multiChunker : multiChunkers) {
            logger.log(Level.INFO, "Running with " + chunker.getClass() + " and " + multiChunker.getClass());
            chunkFileIntoMultiChunks(chunker, multiChunker, minMultiChunkSize);
        }
    }
}