Examples with ChunkChecksum org.syncany.database.ChunkEntry.ChunkChecksum used on opensource projects

Example 11 with ChunkChecksum

use of org.syncany.database.ChunkEntry.ChunkChecksum in project syncany by syncany.

the class FrameworkCombinationTest method deduplicateAndCreateChunkIndex.

private ChunkIndex deduplicateAndCreateChunkIndex(final List<File> inputFiles, FrameworkCombination combination) throws IOException {
    logger.log(Level.INFO, "- Deduplicate and create chunk index ...");
    final ChunkIndex chunkIndex = new ChunkIndex();
    Deduper deduper = new Deduper(combination.chunker, combination.multiChunker, combination.transformer, Long.MAX_VALUE, Long.MAX_VALUE);
    deduper.deduplicate(inputFiles, new DeduperListener() {

        @Override
        public void onMultiChunkWrite(MultiChunk multiChunk, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to multichunk " + multiChunk.getId() + " ...");
            chunkIndex.chunkIDToMultiChunkID.put(new ChunkChecksum(chunk.getChecksum()), multiChunk.getId());
        }

        @Override
        public void onFileAddChunk(File file, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to inputFileToChunkIDs-map for file " + file + " ...");
            List<ChunkChecksum> chunkIDsForFile = chunkIndex.inputFileToChunkIDs.get(file);
            if (chunkIDsForFile == null) {
                chunkIDsForFile = new ArrayList<ChunkChecksum>();
            }
            chunkIDsForFile.add(new ChunkChecksum(chunk.getChecksum()));
            chunkIndex.inputFileToChunkIDs.put(file, chunkIDsForFile);
        }

        @Override
        public boolean onChunk(Chunk chunk) {
            if (chunkIndex.chunkIDToMultiChunkID.containsKey(new ChunkChecksum(chunk.getChecksum()))) {
                logger.log(Level.INFO, "  + Known chunk " + StringUtil.toHex(chunk.getChecksum()));
                return false;
            } else {
                logger.log(Level.INFO, "  + New chunk " + StringUtil.toHex(chunk.getChecksum()));
                return true;
            }
        }

        @Override
        public File getMultiChunkFile(MultiChunkId multiChunkId) {
            File outputMultiChunk = new File(tempDir + "/multichunk-" + multiChunkId);
            chunkIndex.outputMultiChunkFiles.add(outputMultiChunk);
            return outputMultiChunk;
        }

        @Override
        public MultiChunkId createNewMultiChunkId(Chunk firstChunk) {
            // Note: In the real implementation, this should be random
            return new MultiChunkId(firstChunk.getChecksum());
        }

        @Override
        public boolean onFileFilter(File file) {
            return true;
        }

        @Override
        public boolean onFileStart(File file) {
            return file.isFile() && !FileUtil.isSymlink(file);
        }

        @Override
        public void onFileEnd(File file, byte[] checksum) {
        // Empty
        }

        @Override
        public void onMultiChunkOpen(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onMultiChunkClose(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onStart(int fileCount) {
        // Empty
        }

        @Override
        public void onFinish() {
        // Empty
        }
    });
    return chunkIndex;
}

Example 12 with ChunkChecksum

use of org.syncany.database.ChunkEntry.ChunkChecksum in project syncany by syncany.

the class FrameworkCombinationTest method testBlackBoxCombination.

private void testBlackBoxCombination(List<File> inputFiles, FrameworkCombination combination) throws Exception {
    // Deduplicate
    ChunkIndex chunkIndex = deduplicateAndCreateChunkIndex(inputFiles, combination);
    // Assemble
    Map<ChunkChecksum, File> extractedChunkIDToChunkFile = extractChunksFromMultiChunks(chunkIndex.outputMultiChunkFiles, combination);
    Map<File, File> inputFilesToReassembledOutputFiles = reassembleFiles(chunkIndex.inputFileToChunkIDs, extractedChunkIDToChunkFile);
    // Compare checksums of files
    for (Map.Entry<File, File> inputFilesToReassembledOutputFilesEntry : inputFilesToReassembledOutputFiles.entrySet()) {
        File inputFile = inputFilesToReassembledOutputFilesEntry.getKey();
        File outputFile = inputFilesToReassembledOutputFilesEntry.getValue();
        byte[] inputFileChecksum = TestFileUtil.createChecksum(inputFile);
        byte[] outputFileChecksum = TestFileUtil.createChecksum(outputFile);
        assertArrayEquals("Input file and output file checksums do not match" + "for files " + inputFile + " and " + outputFile, inputFileChecksum, outputFileChecksum);
    }
}

Example 13 with ChunkChecksum

use of org.syncany.database.ChunkEntry.ChunkChecksum in project syncany by syncany.

the class FrameworkCombinationTest method reassembleFiles.

private Map<File, File> reassembleFiles(Map<File, List<ChunkChecksum>> inputFileToChunkIDs, Map<ChunkChecksum, File> extractedChunkIDToChunkFile) throws IOException {
    Map<File, File> inputFileToOutputFile = new HashMap<File, File>();
    for (Map.Entry<File, List<ChunkChecksum>> inputFileToChunkIDsEntry : inputFileToChunkIDs.entrySet()) {
        File inputFile = inputFileToChunkIDsEntry.getKey();
        List<ChunkChecksum> chunkIDs = inputFileToChunkIDsEntry.getValue();
        File outputFile = new File(tempDir + "/reassembledfile-" + inputFile.getName());
        FileOutputStream outputFileOutputStream = new FileOutputStream(outputFile);
        logger.log(Level.INFO, "- Reassemble file " + inputFile + " to " + outputFile + " ...");
        for (ChunkChecksum chunkID : chunkIDs) {
            File extractedChunkFile = extractedChunkIDToChunkFile.get(chunkID);
            logger.log(Level.INFO, "  + Appending " + chunkID + " (file: " + extractedChunkFile + ") to " + outputFile + " ...");
            IOUtils.copy(new FileInputStream(extractedChunkFile), outputFileOutputStream);
        }
        inputFileToOutputFile.put(inputFile, outputFile);
    }
    return inputFileToOutputFile;
}

Example 14 with ChunkChecksum

use of org.syncany.database.ChunkEntry.ChunkChecksum in project syncany by syncany.

the class DatabaseXmlParseHandler method startElement.

@Override
public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
    elementPath += "/" + qName;
    if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion")) {
        databaseVersion = new DatabaseVersion();
    } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/header/time")) {
        Date timeValue = new Date(Long.parseLong(attributes.getValue("value")));
        databaseVersion.setTimestamp(timeValue);
    } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/header/client")) {
        String clientName = attributes.getValue("name");
        databaseVersion.setClient(clientName);
    } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/header/vectorClock")) {
        vectorClock = new VectorClock();
    } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/header/vectorClock/client")) {
        String clientName = attributes.getValue("name");
        Long clientValue = Long.parseLong(attributes.getValue("value"));
        vectorClock.setClock(clientName, clientValue);
    } else if (readType == DatabaseReadType.FULL) {
        if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/chunks/chunk")) {
            String chunkChecksumStr = attributes.getValue("checksum");
            ChunkChecksum chunkChecksum = ChunkChecksum.parseChunkChecksum(chunkChecksumStr);
            int chunkSize = Integer.parseInt(attributes.getValue("size"));
            ChunkEntry chunkEntry = new ChunkEntry(chunkChecksum, chunkSize);
            databaseVersion.addChunk(chunkEntry);
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/fileContents/fileContent")) {
            String checksumStr = attributes.getValue("checksum");
            long size = Long.parseLong(attributes.getValue("size"));
            fileContent = new FileContent();
            fileContent.setChecksum(FileChecksum.parseFileChecksum(checksumStr));
            fileContent.setSize(size);
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/fileContents/fileContent/chunkRefs/chunkRef")) {
            String chunkChecksumStr = attributes.getValue("ref");
            fileContent.addChunk(ChunkChecksum.parseChunkChecksum(chunkChecksumStr));
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/multiChunks/multiChunk")) {
            String multChunkIdStr = attributes.getValue("id");
            MultiChunkId multiChunkId = MultiChunkId.parseMultiChunkId(multChunkIdStr);
            long size = Long.parseLong(attributes.getValue("size"));
            if (multiChunkId == null) {
                throw new SAXException("Cannot read ID from multichunk " + multChunkIdStr);
            }
            multiChunk = new MultiChunkEntry(multiChunkId, size);
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/multiChunks/multiChunk/chunkRefs/chunkRef")) {
            String chunkChecksumStr = attributes.getValue("ref");
            multiChunk.addChunk(ChunkChecksum.parseChunkChecksum(chunkChecksumStr));
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/fileHistories/fileHistory")) {
            String fileHistoryIdStr = attributes.getValue("id");
            FileHistoryId fileId = FileHistoryId.parseFileId(fileHistoryIdStr);
            fileHistory = new PartialFileHistory(fileId);
        } else if (elementPath.equalsIgnoreCase("/database/databaseVersions/databaseVersion/fileHistories/fileHistory/fileVersions/fileVersion")) {
            String fileVersionStr = attributes.getValue("version");
            String path = attributes.getValue("path");
            String pathEncoded = attributes.getValue("pathEncoded");
            String sizeStr = attributes.getValue("size");
            String typeStr = attributes.getValue("type");
            String statusStr = attributes.getValue("status");
            String lastModifiedStr = attributes.getValue("lastModified");
            String updatedStr = attributes.getValue("updated");
            String checksumStr = attributes.getValue("checksum");
            String linkTarget = attributes.getValue("linkTarget");
            String dosAttributes = attributes.getValue("dosattrs");
            String posixPermissions = attributes.getValue("posixperms");
            if (fileVersionStr == null || (path == null && pathEncoded == null) || typeStr == null || statusStr == null || sizeStr == null || lastModifiedStr == null) {
                throw new SAXException("FileVersion: Attributes missing: version, path/pathEncoded, type, status, size and last modified are mandatory");
            }
            // Filter it if it was purged somewhere in the future, see #58
            Long fileVersionNum = Long.parseLong(fileVersionStr);
            // Go add it!
            FileVersion fileVersion = new FileVersion();
            fileVersion.setVersion(fileVersionNum);
            if (path != null) {
                fileVersion.setPath(path);
            } else {
                try {
                    fileVersion.setPath(new String(Base64.decodeBase64(pathEncoded), "UTF-8"));
                } catch (UnsupportedEncodingException e) {
                    throw new RuntimeException("Invalid Base64 encoding for filename: " + pathEncoded);
                }
            }
            fileVersion.setType(FileType.valueOf(typeStr));
            fileVersion.setStatus(FileStatus.valueOf(statusStr));
            fileVersion.setSize(Long.parseLong(sizeStr));
            fileVersion.setLastModified(new Date(Long.parseLong(lastModifiedStr)));
            if (updatedStr != null) {
                fileVersion.setUpdated(new Date(Long.parseLong(updatedStr)));
            }
            if (checksumStr != null) {
                fileVersion.setChecksum(FileChecksum.parseFileChecksum(checksumStr));
            }
            if (linkTarget != null) {
                fileVersion.setLinkTarget(linkTarget);
            }
            if (dosAttributes != null) {
                fileVersion.setDosAttributes(dosAttributes);
            }
            if (posixPermissions != null) {
                fileVersion.setPosixPermissions(posixPermissions);
            }
            fileHistory.addFileVersion(fileVersion);
        }
    }
}

Example 15 with ChunkChecksum

use of org.syncany.database.ChunkEntry.ChunkChecksum in project syncany by syncany.

the class MultiChunkSqlDao method writeMultiChunkRefs.

private void writeMultiChunkRefs(Connection connection, MultiChunkEntry multiChunk) throws SQLException {
    PreparedStatement preparedStatement = getStatement("multichunk.insert.all.writeMultiChunkRefs.sql");
    for (ChunkChecksum chunkChecksum : multiChunk.getChunks()) {
        preparedStatement.setString(1, multiChunk.getId().toString());
        preparedStatement.setString(2, chunkChecksum.toString());
        preparedStatement.addBatch();
    }
    preparedStatement.executeBatch();
    preparedStatement.close();
}