Examples with Chunker org.syncany.chunk.Chunker used on opensource projects

Example 1 with Chunker

use of org.syncany.chunk.Chunker in project syncany by syncany.

the class Assembler method assembleToCache.

/**
 * Assembles the given file version to the local cache and returns a reference
 * to the cached file after successfully assembling the file.
 */
public File assembleToCache(FileVersion fileVersion) throws Exception {
    File reconstructedFileInCache = config.getCache().createTempFile("reconstructedFileVersion");
    logger.log(Level.INFO, "     - Creating file " + fileVersion.getPath() + " to " + reconstructedFileInCache + " ...");
    FileContent fileContent = localDatabase.getFileContent(fileVersion.getChecksum(), true);
    if (fileContent == null && memoryDatabase != null) {
        fileContent = memoryDatabase.getContent(fileVersion.getChecksum());
    }
    // Check consistency!
    if (fileContent == null && fileVersion.getChecksum() != null) {
        throw new Exception("Cannot determine file content for checksum " + fileVersion.getChecksum());
    }
    // Create empty file
    if (fileContent == null) {
        FileUtils.touch(reconstructedFileInCache);
        return reconstructedFileInCache;
    }
    // Create non-empty file
    Chunker chunker = config.getChunker();
    MultiChunker multiChunker = config.getMultiChunker();
    FileOutputStream reconstructedFileOutputStream = new FileOutputStream(reconstructedFileInCache);
    MessageDigest reconstructedFileChecksum = MessageDigest.getInstance(chunker.getChecksumAlgorithm());
    if (fileContent != null) {
        // File can be empty!
        Collection<ChunkChecksum> fileChunks = fileContent.getChunks();
        for (ChunkChecksum chunkChecksum : fileChunks) {
            MultiChunkId multiChunkIdForChunk = localDatabase.getMultiChunkId(chunkChecksum);
            if (multiChunkIdForChunk == null && memoryDatabase != null) {
                multiChunkIdForChunk = memoryDatabase.getMultiChunkIdForChunk(chunkChecksum);
            }
            File decryptedMultiChunkFile = config.getCache().getDecryptedMultiChunkFile(multiChunkIdForChunk);
            MultiChunk multiChunk = multiChunker.createMultiChunk(decryptedMultiChunkFile);
            InputStream chunkInputStream = multiChunk.getChunkInputStream(chunkChecksum.getBytes());
            byte[] buffer = new byte[4096];
            int read = 0;
            while (-1 != (read = chunkInputStream.read(buffer))) {
                reconstructedFileChecksum.update(buffer, 0, read);
                reconstructedFileOutputStream.write(buffer, 0, read);
            }
            chunkInputStream.close();
            multiChunk.close();
        }
    }
    reconstructedFileOutputStream.close();
    // Validate checksum
    byte[] reconstructedFileExpectedChecksum = fileContent.getChecksum().getBytes();
    byte[] reconstructedFileActualChecksum = reconstructedFileChecksum.digest();
    if (!Arrays.equals(reconstructedFileActualChecksum, reconstructedFileExpectedChecksum)) {
        throw new Exception("Checksums do not match: expected " + StringUtil.toHex(reconstructedFileExpectedChecksum) + " != actual " + StringUtil.toHex(reconstructedFileActualChecksum));
    }
    return reconstructedFileInCache;
}

Example 2 with Chunker

use of org.syncany.chunk.Chunker in project syncany by syncany.

the class FrameworkCombinationTest method fillCombinations.

private void fillCombinations() throws InvalidKeyException, NoSuchAlgorithmException, NoSuchPaddingException {
    // MultiChunks
    List<MultiChunker> multiChunkers = new LinkedList<MultiChunker>();
    int[] multiChunkSizes = new int[] { 250000, 500000 };
    for (int i = 0; i < multiChunkSizes.length; i++) {
        // multiChunkers.add(new CustomMultiChunker(multiChunkSizes[i]));
        multiChunkers.add(new ZipMultiChunker(multiChunkSizes[i]));
    }
    // Chunks
    List<Chunker> chunkers = new LinkedList<Chunker>();
    int[] chunkSizes = new int[] { 8000, 16000 };
    String[] digestAlgs = new String[] { /*"MD5" ,*/
    "SHA1" };
    String[] fingerprinters = new String[] { "Adler32" /*, "Plain", "Rabin"*/
    };
    for (int i = 0; i < chunkSizes.length; i++) {
        for (int j = 0; j < digestAlgs.length; j++) {
            for (int k = 0; k < fingerprinters.length; k++) {
                chunkers.add(new TttdChunker(chunkSizes[i], TttdChunker.DEFAULT_WINDOW_SIZE, digestAlgs[j], fingerprinters[k]));
            }
        }
    }
    // Compression/Encryption
    List<CipherSpec> cipherSpecs = new ArrayList<CipherSpec>();
    cipherSpecs.add(CipherSpecs.getCipherSpec(1));
    cipherSpecs.add(CipherSpecs.getCipherSpec(2));
    List<Transformer> transformerChains = new LinkedList<Transformer>();
    transformerChains.add(new NoTransformer());
    transformerChains.add(new GzipTransformer());
    transformerChains.add(new CipherTransformer(cipherSpecs, masterKey));
    transformerChains.add(new GzipTransformer(new CipherTransformer(cipherSpecs, masterKey)));
    for (MultiChunker multiChunker : multiChunkers) {
        for (Transformer transformer : transformerChains) {
            for (Chunker chunker : chunkers) {
                String configName = multiChunker + "/" + chunker + "/" + transformer;
                combinations.add(new FrameworkCombination(configName, chunker, multiChunker, transformer));
            }
        }
    }
}

Example 3 with Chunker

use of org.syncany.chunk.Chunker in project syncany by syncany.

the class TTTDChunkerTest method testNextChunkEvenIfThereAreNone.

@Test
public void testNextChunkEvenIfThereAreNone() throws IOException {
    // Test Constants
    final int TOTAL_FILE_SIZE = 5 * 1024;
    final int CHUNK_SIZE = 512 * 1024;
    // Setup
    File inputFile = TestFileUtil.createRandomFileInDirectory(tempDir, TOTAL_FILE_SIZE);
    Chunker chunker = new TttdChunker(CHUNK_SIZE);
    // Create chunks
    Enumeration<Chunk> chunkEnumeration = chunker.createChunks(inputFile);
    while (chunkEnumeration.hasMoreElements()) {
        chunkEnumeration.nextElement();
    }
    // This should lead to an IOException
    assertNull("No chunk expected, but data received.", chunkEnumeration.nextElement());
    assertFalse("hasElements() should return 'false' if no chunk available.", chunkEnumeration.hasMoreElements());
}

Example 4 with Chunker

use of org.syncany.chunk.Chunker in project syncany by syncany.

the class FixedOffsetChunkerTest method testCreateChunksFrom5MBFileAndTestChunkSize.

@Test
public void testCreateChunksFrom5MBFileAndTestChunkSize() throws Exception {
    // Test Constants
    final int TOTAL_FILE_SIZE = 5 * 1024 * 1024;
    final int EXACT_CHUNK_SIZE = 512 * 1024;
    final int EXPECTED_NUMBER_OF_CHUNKS = TOTAL_FILE_SIZE / EXACT_CHUNK_SIZE;
    final int EXPECTED_CHUNK_SIZE = EXACT_CHUNK_SIZE;
    // Setup
    File inputRandom5MBFile = TestFileUtil.createRandomFileInDirectory(tempDir, TOTAL_FILE_SIZE);
    File outputCopyOfRandom5MBFile = TestFileUtil.getRandomFilenameInDirectory(tempDir);
    FileOutputStream outputCopyOfRandom5MBFileOutputStream = new FileOutputStream(outputCopyOfRandom5MBFile);
    Chunker chunker = new FixedChunker(EXACT_CHUNK_SIZE, FixedChunker.DEFAULT_DIGEST_ALG);
    // Create chunks
    int actualChunkCount = 0;
    Enumeration<Chunk> chunkEnumeration = chunker.createChunks(inputRandom5MBFile);
    Chunk lastChunk = null;
    while (chunkEnumeration.hasMoreElements()) {
        actualChunkCount++;
        lastChunk = chunkEnumeration.nextElement();
        // Chunk size & checksum
        assertEquals("Chunk does not have the expected size.", EXPECTED_CHUNK_SIZE, lastChunk.getSize());
        assertNotNull("Chunk checksum should not be null.", lastChunk.getChecksum());
        outputCopyOfRandom5MBFileOutputStream.write(lastChunk.getContent());
    }
    outputCopyOfRandom5MBFileOutputStream.close();
    // Number of chunks
    assertEquals("Unexpected number of chunks when chunking", EXPECTED_NUMBER_OF_CHUNKS, actualChunkCount);
    // Checksums
    byte[] inputFileChecksum = FileUtil.createChecksum(inputRandom5MBFile, FixedChunker.DEFAULT_DIGEST_ALG);
    byte[] outputFileChecksum = FileUtil.createChecksum(outputCopyOfRandom5MBFile, FixedChunker.DEFAULT_DIGEST_ALG);
    assertArrayEquals("Checksums of input and output file do not match.", inputFileChecksum, outputFileChecksum);
    assertArrayEquals("Last chunk's getFileChecksum() should be the file checksum.", inputFileChecksum, lastChunk.getFileChecksum());
}

Example 5 with Chunker

use of org.syncany.chunk.Chunker in project syncany by syncany.

the class TTTDChunkerTest method testStringSerialization.

@Test
public void testStringSerialization() {
    final int CHUNK_SIZE = 512 * 1024;
    Chunker chunker = new TttdChunker(CHUNK_SIZE);
    assertEquals("Other toString() result expected.", "TTTD-" + CHUNK_SIZE + "-" + TttdChunker.DEFAULT_DIGEST_ALG + "-" + TttdChunker.DEFAULT_FINGERPRINT_ALG, chunker.toString());
}