Examples with PartitionTokenScanner org.eclipse.xtext.ui.editor.model.PartitionTokenScanner used on opensource projects

Example 1 with PartitionTokenScanner

use of org.eclipse.xtext.ui.editor.model.PartitionTokenScanner in project xtext-eclipse by eclipse.

the class DocumentPartitionerTest method getDocument.

public XtextDocument getDocument(String s) {
    TerminalsTokenTypeToPartitionMapper mapper = new TerminalsTokenTypeToPartitionMapper() {

        {
            setTokenDefProvider(new AntlrTokenDefProvider() {

                {
                    setAntlrTokenFileProvider(new XtextAntlrTokenFileProvider());
                }
            });
        }
    };
    PartitionTokenScanner scanner = new PartitionTokenScanner();
    scanner.setMapper(mapper);
    DocumentPartitioner partitioner = new DocumentPartitioner(scanner, mapper);
    DocumentTokenSource tokenSource = new DocumentTokenSource();
    tokenSource.setLexer(new Provider<Lexer>() {

        @Override
        public Lexer get() {
            return new org.eclipse.xtext.parser.antlr.internal.InternalXtextLexer();
        }
    });
    XtextDocument document = new XtextDocument(tokenSource, null, new OutdatedStateManager(), new OperationCanceledManager());
    document.setDocumentPartitioner(partitioner);
    partitioner.connect(document);
    document.set(s);
    return document;
}

Example 2 with PartitionTokenScanner

use of org.eclipse.xtext.ui.editor.model.PartitionTokenScanner in project xtext-eclipse by eclipse.

the class PartitionTokenScannerTest method testWholePart.

@Test
public void testWholePart() throws Exception {
    PartitionTokenScanner scanner = getPartitionTokenScanner(t(2, 3), t(4, 3), t(2, 1), t(34, 3));
    scanner.setPartialRange(null, 0, 42, "3", 0);
    assertEquals("3", scanner.nextToken().getData());
    assertEquals(0, scanner.getTokenOffset());
    assertEquals(6, scanner.getTokenLength());
    assertEquals("1", scanner.nextToken().getData());
    assertEquals(6, scanner.getTokenOffset());
    assertEquals(2, scanner.getTokenLength());
    assertEquals("3", scanner.nextToken().getData());
    assertEquals(8, scanner.getTokenOffset());
    assertEquals(34, scanner.getTokenLength());
    assertSame(Token.EOF, scanner.nextToken());
}

Example 3 with PartitionTokenScanner

use of org.eclipse.xtext.ui.editor.model.PartitionTokenScanner in project xtext-eclipse by eclipse.

the class PartitionTokenScannerTest method getPartitionTokenScanner.

public PartitionTokenScanner getPartitionTokenScanner(ILexerTokenRegion... tokenDescs) throws Exception {
    final List<ILexerTokenRegion> tokens = Arrays.asList(tokenDescs);
    int offset = 0;
    for (ILexerTokenRegion token : tokens) {
        ((LexerTokenRegion) token).setOffset(offset);
        offset += token.getLength();
    }
    PartitionTokenScanner tokenScanner = new PartitionTokenScanner() {

        @Override
        protected Iterable<ILexerTokenRegion> getTokens(IDocument document) {
            return tokens;
        }

        @Override
        protected boolean shouldMergePartitions(String contentType) {
            return "3".equals(contentType);
        }
    };
    tokenScanner.setMapper(new ITokenTypeToPartitionTypeMapper() {

        @Override
        public String[] getSupportedPartitionTypes() {
            throw new UnsupportedOperationException();
        }

        @Override
        public String getPartitionType(int antlrTokenType) {
            return "" + antlrTokenType;
        }
    });
    return tokenScanner;
}

Example 4 with PartitionTokenScanner

use of org.eclipse.xtext.ui.editor.model.PartitionTokenScanner in project xtext-eclipse by eclipse.

the class PartitionTokenScannerTest method testMiddlePart.

@Test
public void testMiddlePart() throws Exception {
    PartitionTokenScanner scanner = getPartitionTokenScanner(t(2, 3), t(4, 3), t(2, 1), t(34, 3));
    scanner.setPartialRange(null, 6, 2, "1", 6);
    assertEquals("1", scanner.nextToken().getData());
    assertEquals(6, scanner.getTokenOffset());
    assertEquals(2, scanner.getTokenLength());
    assertSame(Token.EOF, scanner.nextToken());
}

Example 5 with PartitionTokenScanner

use of org.eclipse.xtext.ui.editor.model.PartitionTokenScanner in project xtext-eclipse by eclipse.

the class AbstractXtextDocumentTest method getDocument.

public XtextDocument getDocument(String s) {
    TerminalsTokenTypeToPartitionMapper mapper = new TerminalsTokenTypeToPartitionMapper() {

        {
            setTokenDefProvider(new AntlrTokenDefProvider() {

                {
                    setAntlrTokenFileProvider(new XtextAntlrTokenFileProvider());
                }
            });
        }
    };
    PartitionTokenScanner scanner = new PartitionTokenScanner();
    scanner.setMapper(mapper);
    DocumentPartitioner partitioner = new DocumentPartitioner(scanner, mapper);
    DocumentTokenSource tokenSource = new DocumentTokenSource();
    tokenSource.setLexer(new Provider<Lexer>() {

        @Override
        public Lexer get() {
            return new org.eclipse.xtext.parser.antlr.internal.InternalXtextLexer();
        }
    });
    XtextDocument document = new XtextDocument(tokenSource, null, outdatedStateManager, operationCanceledManager);
    document.setDocumentPartitioner(partitioner);
    partitioner.connect(document);
    document.set(s);
    return document;
}