Examples with TokenChunker org.sonar.duplications.token.TokenChunker used on opensource projects

Example 1 with TokenChunker

use of org.sonar.duplications.token.TokenChunker in project sonarqube by SonarSource.

the class JavaCpdBlockIndexer method createIndex.

private void createIndex(Iterable<InputFile> sourceFiles) {
    TokenChunker tokenChunker = JavaTokenProducer.build();
    StatementChunker statementChunker = JavaStatementBuilder.build();
    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
    for (InputFile inputFile : sourceFiles) {
        LOG.debug("Populating index from {}", inputFile);
        String resourceEffectiveKey = ((DefaultInputFile) inputFile).key();
        List<Statement> statements;
        try (InputStream is = new FileInputStream(inputFile.file());
            Reader reader = new InputStreamReader(is, fs.encoding())) {
            statements = statementChunker.chunk(tokenChunker.chunk(reader));
        } catch (FileNotFoundException e) {
            throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
        } catch (IOException e) {
            throw new IllegalStateException("Exception handling file: " + inputFile.file(), e);
        }
        List<Block> blocks = blockChunker.chunk(resourceEffectiveKey, statements);
        index.insert(inputFile, blocks);
    }
}

Example 2 with TokenChunker

use of org.sonar.duplications.token.TokenChunker in project sonarqube by SonarSource.

the class JavaCpdBlockIndexerSensor method createIndex.

private void createIndex(Iterable<InputFile> sourceFiles) {
    TokenChunker tokenChunker = JavaTokenProducer.build();
    StatementChunker statementChunker = JavaStatementBuilder.build();
    BlockChunker blockChunker = new BlockChunker(BLOCK_SIZE);
    for (InputFile inputFile : sourceFiles) {
        LOG.debug("Populating index from {}", inputFile);
        String resourceEffectiveKey = inputFile.key();
        List<Statement> statements;
        try (InputStream is = inputFile.inputStream();
            Reader reader = new InputStreamReader(is, inputFile.charset())) {
            statements = statementChunker.chunk(tokenChunker.chunk(reader));
        } catch (FileNotFoundException e) {
            throw new IllegalStateException("Cannot find file " + inputFile.file(), e);
        } catch (IOException e) {
            throw new IllegalStateException("Exception handling file: " + inputFile.file(), e);
        }
        List<Block> blocks;
        try {
            blocks = blockChunker.chunk(resourceEffectiveKey, statements);
        } catch (Exception e) {
            throw new IllegalStateException("Cannot process file " + inputFile.file(), e);
        }
        index.insert(inputFile, blocks);
    }
}