Examples with DeduperListener org.syncany.chunk.DeduperListener used on opensource projects

Example 1 with DeduperListener

use of org.syncany.chunk.DeduperListener in project syncany by syncany.

the class FrameworkCombinationTest method deduplicateAndCreateChunkIndex.

private ChunkIndex deduplicateAndCreateChunkIndex(final List<File> inputFiles, FrameworkCombination combination) throws IOException {
    logger.log(Level.INFO, "- Deduplicate and create chunk index ...");
    final ChunkIndex chunkIndex = new ChunkIndex();
    Deduper deduper = new Deduper(combination.chunker, combination.multiChunker, combination.transformer, Long.MAX_VALUE, Long.MAX_VALUE);
    deduper.deduplicate(inputFiles, new DeduperListener() {

        @Override
        public void onMultiChunkWrite(MultiChunk multiChunk, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to multichunk " + multiChunk.getId() + " ...");
            chunkIndex.chunkIDToMultiChunkID.put(new ChunkChecksum(chunk.getChecksum()), multiChunk.getId());
        }

        @Override
        public void onFileAddChunk(File file, Chunk chunk) {
            logger.log(Level.INFO, "    - Adding chunk " + StringUtil.toHex(chunk.getChecksum()) + " to inputFileToChunkIDs-map for file " + file + " ...");
            List<ChunkChecksum> chunkIDsForFile = chunkIndex.inputFileToChunkIDs.get(file);
            if (chunkIDsForFile == null) {
                chunkIDsForFile = new ArrayList<ChunkChecksum>();
            }
            chunkIDsForFile.add(new ChunkChecksum(chunk.getChecksum()));
            chunkIndex.inputFileToChunkIDs.put(file, chunkIDsForFile);
        }

        @Override
        public boolean onChunk(Chunk chunk) {
            if (chunkIndex.chunkIDToMultiChunkID.containsKey(new ChunkChecksum(chunk.getChecksum()))) {
                logger.log(Level.INFO, "  + Known chunk " + StringUtil.toHex(chunk.getChecksum()));
                return false;
            } else {
                logger.log(Level.INFO, "  + New chunk " + StringUtil.toHex(chunk.getChecksum()));
                return true;
            }
        }

        @Override
        public File getMultiChunkFile(MultiChunkId multiChunkId) {
            File outputMultiChunk = new File(tempDir + "/multichunk-" + multiChunkId);
            chunkIndex.outputMultiChunkFiles.add(outputMultiChunk);
            return outputMultiChunk;
        }

        @Override
        public MultiChunkId createNewMultiChunkId(Chunk firstChunk) {
            // Note: In the real implementation, this should be random
            return new MultiChunkId(firstChunk.getChecksum());
        }

        @Override
        public boolean onFileFilter(File file) {
            return true;
        }

        @Override
        public boolean onFileStart(File file) {
            return file.isFile() && !FileUtil.isSymlink(file);
        }

        @Override
        public void onFileEnd(File file, byte[] checksum) {
        // Empty
        }

        @Override
        public void onMultiChunkOpen(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onMultiChunkClose(MultiChunk multiChunk) {
        // Empty
        }

        @Override
        public void onStart(int fileCount) {
        // Empty
        }

        @Override
        public void onFinish() {
        // Empty
        }
    });
    return chunkIndex;
}

Example 2 with DeduperListener

use of org.syncany.chunk.DeduperListener in project syncany by syncany.

the class Indexer method indexWithNewFiles.

private void indexWithNewFiles(List<File> files, List<File> deletedFiles, Queue<DatabaseVersion> databaseVersionQueue) throws IOException {
    boolean isFirstFile = true;
    int filesCount = files.size();
    while (!files.isEmpty()) {
        DatabaseVersion newDatabaseVersion = new DatabaseVersion();
        // Create the DeduperListener that will receive MultiChunks and store them in the DatabaseVersion object
        DeduperListener deduperListener = new IndexerDeduperListener(newDatabaseVersion);
        // Signal the start of indexing if we are about to deduplicate the first file
        if (isFirstFile) {
            deduperListener.onStart(files.size());
            // Add deletions in first database version
            removeDeletedFiles(newDatabaseVersion, deletedFiles);
            isFirstFile = false;
        }
        // Find and index new files
        deduper.deduplicate(files, deduperListener);
        if (!newDatabaseVersion.getFileHistories().isEmpty()) {
            logger.log(Level.FINE, "Processed new database version: " + newDatabaseVersion);
            databaseVersionQueue.offer(newDatabaseVersion);
            int remainingFilesCount = filesCount - files.size();
            eventBus.post(new UpIndexMidSyncExternalEvent(config.getLocalDir().toString(), filesCount, remainingFilesCount));
        }
    // else { (comment-only else case)
    // Just chunks and multichunks, no filehistory. Since this means the file was being
    // written/vanished during operations, it makes no sense to upload it. If the user
    // wants it indexed, Up can be run again.
    // }
    }
}