Examples with Annotation com.joliciel.talismane.Annotation used on opensource projects

Example 6 with Annotation

use of com.joliciel.talismane.Annotation in project talismane by joliciel-informatique.

the class NewlineEndOfSentenceMarkerTest method testApply.

@Test
public void testApply() throws Exception {
    NewlineEndOfSentenceMarker filter = new NewlineEndOfSentenceMarker(1000);
    AnnotatedText text = new AnnotatedText("1\r\n2\r\n");
    filter.annotate(text);
    LOG.debug(text.getAnnotations().toString());
    List<Annotation<RawTextSentenceBreakMarker>> sentenceBreaks = text.getAnnotations(RawTextSentenceBreakMarker.class);
    assertEquals(2, sentenceBreaks.size());
    List<Annotation<RawTextSkipMarker>> skips = text.getAnnotations(RawTextSkipMarker.class);
    assertEquals(2, skips.size());
    assertEquals(1, sentenceBreaks.get(0).getStart());
    assertEquals(3, sentenceBreaks.get(0).getEnd());
    assertEquals(1, skips.get(0).getStart());
    assertEquals(3, skips.get(0).getEnd());
    assertEquals(4, sentenceBreaks.get(1).getStart());
    assertEquals(6, sentenceBreaks.get(1).getEnd());
    assertEquals(4, skips.get(1).getStart());
    assertEquals(6, skips.get(1).getEnd());
    text = new AnnotatedText("1\r2\r");
    filter.annotate(text);
    LOG.debug(text.getAnnotations().toString());
    sentenceBreaks = text.getAnnotations(RawTextSentenceBreakMarker.class);
    assertEquals(2, sentenceBreaks.size());
    skips = text.getAnnotations(RawTextSkipMarker.class);
    assertEquals(2, skips.size());
    assertEquals(1, sentenceBreaks.get(0).getStart());
    assertEquals(2, sentenceBreaks.get(0).getEnd());
    assertEquals(1, skips.get(0).getStart());
    assertEquals(2, skips.get(0).getEnd());
    assertEquals(3, sentenceBreaks.get(1).getStart());
    assertEquals(4, sentenceBreaks.get(1).getEnd());
    assertEquals(3, skips.get(1).getStart());
    assertEquals(4, skips.get(1).getEnd());
    text = new AnnotatedText("1\r2\r");
    filter.annotate(text);
    LOG.debug(text.getAnnotations().toString());
    sentenceBreaks = text.getAnnotations(RawTextSentenceBreakMarker.class);
    assertEquals(2, sentenceBreaks.size());
    skips = text.getAnnotations(RawTextSkipMarker.class);
    assertEquals(2, skips.size());
    assertEquals(1, sentenceBreaks.get(0).getStart());
    assertEquals(2, sentenceBreaks.get(0).getEnd());
    assertEquals(1, skips.get(0).getStart());
    assertEquals(2, skips.get(0).getEnd());
    assertEquals(3, sentenceBreaks.get(1).getStart());
    assertEquals(4, sentenceBreaks.get(1).getEnd());
    assertEquals(3, skips.get(1).getStart());
    assertEquals(4, skips.get(1).getEnd());
}

Example 7 with Annotation

use of com.joliciel.talismane.Annotation in project talismane by joliciel-informatique.

the class RawTextTest method testGetProcessedText.

@Test
public void testGetProcessedText() throws Exception {
    System.setProperty("config.file", "src/test/resources/test.conf");
    ConfigFactory.invalidateCaches();
    final Config config = ConfigFactory.load();
    final String sessionId = "test";
    String[] labels = new String[0];
    String text = "1 2 3<skip>skip</skip> 4<skip>skip</skip> five";
    RawText rawText = new RawText(text, true, sessionId);
    List<Annotation<RawTextSkipMarker>> skips = new ArrayList<>();
    skips.add(new Annotation<>("1 2 3".length(), "1 2 3<skip>skip</skip>".length(), new RawTextSkipMarker("me"), labels));
    skips.add(new Annotation<>("1 2 3<skip>skip</skip> 4".length(), "1 2 3<skip>skip</skip> 4<skip>skip</skip>".length(), new RawTextSkipMarker("me"), labels));
    rawText.addAnnotations(skips);
    List<Annotation<RawTextReplaceMarker>> replaces = new ArrayList<>();
    replaces.add(new Annotation<>("1 2 3<skip>skip</skip> 4<skip>skip</skip> ".length(), "1 2 3<skip>skip</skip> 4<skip>skip</skip> five".length(), new RawTextReplaceMarker("me", "5"), labels));
    rawText.addAnnotations(replaces);
    AnnotatedText processedTextBlock = rawText.getProcessedText();
    assertEquals("1 2 3 4 5", processedTextBlock.getText());
}

Example 8 with Annotation

use of com.joliciel.talismane.Annotation in project talismane by joliciel-informatique.

the class RawTextTest method testNoSentenceAnnotationLocation.

@Test
public void testNoSentenceAnnotationLocation() throws Exception {
    System.setProperty("config.file", "src/test/resources/test.conf");
    ConfigFactory.invalidateCaches();
    final Config config = ConfigFactory.load();
    final String sessionId = "test";
    String[] labels = new String[0];
    String text = "Mr. Jones and <skip/>Mrs. Smith.";
    RawText textBlock = new RawText(text, true, sessionId);
    List<Annotation<RawTextNoSentenceBreakMarker>> noSentenceBreaks = new ArrayList<>();
    System.out.println("we add no sentence break annotations (as if they were added by a filter)");
    noSentenceBreaks.add(new Annotation<>("".length(), "Mr.".length(), new RawTextNoSentenceBreakMarker("me"), labels));
    noSentenceBreaks.add(new Annotation<>("Mr. Jones and <skip/>".length(), "Mr. Jones and <skip/>Mrs.".length(), new RawTextNoSentenceBreakMarker("me"), labels));
    textBlock.addAnnotations(noSentenceBreaks);
    List<Annotation<RawTextSkipMarker>> skips = new ArrayList<>();
    skips.add(new Annotation<>("Mr. Jones and ".length(), "Mr. Jones and <skip/>".length(), new RawTextSkipMarker("me"), labels));
    textBlock.addAnnotations(skips);
    System.out.println("textBlock text: " + textBlock.getText());
    System.out.println("textBlock annotations: " + textBlock.getAnnotations().toString());
    AnnotatedText processedTextBlock = textBlock.getProcessedText();
    assertEquals("Mr. Jones and Mrs. Smith.", processedTextBlock.getText());
    // ensure that the no sentence break text got added at the right place
    // in the processed text
    noSentenceBreaks = processedTextBlock.getAnnotations(RawTextNoSentenceBreakMarker.class);
    System.out.println("Processed annotations: " + noSentenceBreaks);
    assertEquals(2, noSentenceBreaks.size());
    assertEquals("".length(), noSentenceBreaks.get(0).getStart());
    assertEquals("Mr.".length(), noSentenceBreaks.get(0).getEnd());
    assertEquals("Mr. Jones and ".length(), noSentenceBreaks.get(1).getStart());
    assertEquals("Mr. Jones and Mrs.".length(), noSentenceBreaks.get(1).getEnd());
}

Example 9 with Annotation

use of com.joliciel.talismane.Annotation in project talismane by joliciel-informatique.

the class RegexMarkerFilterTest method testUnaryOperatorsStart.

@Test
public void testUnaryOperatorsStart() throws Exception {
    RawTextRegexAnnotator filter = new RawTextRegexAnnotator(RawTextMarkType.START, "<skip>", 0, 1000);
    AnnotatedText text = new AnnotatedText("J'ai du <skip>skip me</skip>mal à le croire.<skip>skip this</skip>");
    filter.annotate(text);
    LOG.debug(text.getAnnotations().toString());
    List<Annotation<RawTextMarker>> markers = text.getAnnotations(RawTextMarker.class);
    assertEquals(2, markers.size());
    int i = 0;
    for (Annotation<RawTextMarker> textMarker : markers) {
        if (i == 0) {
            assertEquals(RawTextMarkType.START, textMarker.getData().getType());
            assertEquals("J'ai du ".length(), textMarker.getStart());
            assertEquals("J'ai du <skip>".length(), textMarker.getEnd());
        } else if (i == 1) {
            assertEquals(RawTextMarkType.START, textMarker.getData().getType());
            assertEquals("J'ai du <skip>skip me</skip>mal à le croire.".length(), textMarker.getStart());
            assertEquals("J'ai du <skip>skip me</skip>mal à le croire.<skip>".length(), textMarker.getEnd());
        }
        i++;
    }
}

Example 10 with Annotation

use of com.joliciel.talismane.Annotation in project talismane by joliciel-informatique.

the class RegexMarkerFilterTest method testApplyWithGroup.

@Test
public void testApplyWithGroup() throws Exception {
    RawTextRegexAnnotator filter = new RawTextRegexAnnotator(RawTextMarkType.SKIP, "<skip>(.*?)</skip>", 1, 1000);
    AnnotatedText text = new AnnotatedText("J'ai du <skip>skip me</skip>mal à le croire.<skip>skip me</skip>");
    filter.annotate(text);
    LOG.debug(text.getAnnotations().toString());
    List<Annotation<RawTextSkipMarker>> skips = text.getAnnotations(RawTextSkipMarker.class);
    assertEquals(2, skips.size());
    int i = 0;
    for (Annotation<RawTextSkipMarker> skip : skips) {
        if (i == 0) {
            assertEquals("J'ai du <skip>".length(), skip.getStart());
            assertEquals("J'ai du <skip>skip me".length(), skip.getEnd());
        } else if (i == 2) {
            assertEquals("J'ai du <skip>skip me</skip>mal à le croire.<skip>".length(), skip.getStart());
            assertEquals("J'ai du <skip>skip me</skip>mal à le croire.<skip>skip me".length(), skip.getEnd());
        }
        i++;
    }
}