Examples with TextField edu.uci.ics.texera.api.field.TextField used on opensource projects

Example 56 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class FileSourceOperatorTest method test3.

/*
     * Test FileSourceOperator with a Directory with recursive = true and maxDepth = null.
     * 
     * All the files under the recursive sub-directories will be read.
     *     
     * expected results: test1.txt, test2.txt, test4.txt and test5.txt will be included
     */
@Test
public void test3() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName, true, null);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)), new Tuple(schema, new TextField(tempFile4String)), new Tuple(schema, new TextField(tempFile5String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 57 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class FileSourceOperatorTest method test2.

/*
     * Test FileSourceOperator with a Directory.
     * Optional parameters are all set to default. (only list files directly in this folder)
     * 
     * Only the files directly under this directory will be used.
     *     
     * expected results: test1.txt and test2.txt will be included.
     */
@Test
public void test2() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 58 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class SampleExtraction method parsePromedHTML.

public static Tuple parsePromedHTML(String fileName, String content) {
    try {
        Document parsedDocument = Jsoup.parse(content);
        String mainText = parsedDocument.getElementById("preview").text();
        Tuple tuple = new Tuple(PromedSchema.PROMED_SCHEMA, new StringField(fileName), new TextField(mainText));
        return tuple;
    } catch (Exception e) {
        return null;
    }
}

Example 59 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class TwitterSample method writeTwitterIndex.

public static void writeTwitterIndex() throws Exception {
    RelationManager relationManager = RelationManager.getInstance();
    relationManager.deleteTable(twitterClimateTable);
    relationManager.createTable(twitterClimateTable, Utils.getDefaultIndexDirectory().resolve(twitterClimateTable), TwitterSchema.TWITTER_SCHEMA, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter dataWriter = relationManager.getTableDataWriter(twitterClimateTable);
    dataWriter.open();
    JsonNode jsonNode = new ObjectMapper().readTree(new File(twitterFilePath));
    for (JsonNode tweet : jsonNode) {
        try {
            String text = tweet.get("text").asText();
            Long id = tweet.get("id").asLong();
            String tweetLink = "https://twitter.com/statuses/" + id;
            JsonNode userNode = tweet.get("user");
            String userScreenName = userNode.get("screen_name").asText();
            String userLink = "https://twitter.com/" + userScreenName;
            String userName = userNode.get("name").asText();
            String userDescription = userNode.get("description").asText();
            Integer userFollowersCount = userNode.get("followers_count").asInt();
            Integer userFriendsCount = userNode.get("friends_count").asInt();
            JsonNode geoTagNode = tweet.get("geo_tag");
            String state = geoTagNode.get("stateName").asText();
            String county = geoTagNode.get("countyName").asText();
            String city = geoTagNode.get("cityName").asText();
            String createAt = tweet.get("create_at").asText();
            Tuple tuple = new Tuple(TwitterSchema.TWITTER_SCHEMA, new TextField(text), new StringField(tweetLink), new StringField(userLink), new TextField(userScreenName), new TextField(userName), new TextField(userDescription), new IntegerField(userFollowersCount), new IntegerField(userFriendsCount), new TextField(state), new TextField(county), new TextField(city), new StringField(createAt));
            dataWriter.insertTuple(tuple);
        } catch (Exception e) {
            // continue to next tuple if something goes wrong
            continue;
        }
    }
    dataWriter.close();
}

Example 60 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class TwitterFeedOperator method getNextTuple.

@Override
public Tuple getNextTuple() throws TexeraException {
    if (cursor == CLOSED || resultCursor >= limit - 1 || resultCursor >= predicate.getTweetNum() - 1) {
        return null;
    }
    if (twitterConnector.getClient().isDone()) {
        System.out.println("Client connection closed unexpectedly: " + twitterConnector.getClient().getExitEvent().getMessage());
        return null;
    }
    try {
        msg = twitterConnector.getMsgQueue().poll(timeout, TimeUnit.SECONDS);
        if (msg == null || msg.length() == 0) {
            System.out.println("Did not receive a message in " + timeout + " seconds");
            return null;
        }
        JsonNode tweet = new ObjectMapper().readValue(msg, JsonNode.class);
        sourceTuple = new Tuple(outputSchema, IDField.newRandomID(), new TextField(TwitterUtils.getText(tweet)), new StringField(TwitterUtils.getMediaLink(tweet)), new StringField(TwitterUtils.getTweetLink(tweet)), new StringField(TwitterUtils.getUserLink(tweet)), new TextField(TwitterUtils.getUserScreenName(tweet)), new TextField(TwitterUtils.getUserName(tweet)), new TextField(TwitterUtils.getUserDescription(tweet)), new IntegerField(TwitterUtils.getUserFollowerCnt(tweet)), new IntegerField(TwitterUtils.getUserFriendsCnt(tweet)), new TextField(TwitterUtils.getUserLocation(tweet)), new StringField(TwitterUtils.getCreateTime(tweet)), new TextField(TwitterUtils.getPlaceName(tweet)), new StringField(TwitterUtils.getCoordinates(tweet)), new StringField(TwitterUtils.getLanguage(tweet)));
        resultCursor++;
        return sourceTuple;
    } catch (InterruptedException e) {
        System.out.println(e);
    } catch (IOException e) {
        e.printStackTrace();
    }
    return null;
}