Examples with DataWriter edu.uci.ics.textdb.storage.DataWriter used on opensource projects

Example 31 with DataWriter

use of edu.uci.ics.textdb.storage.DataWriter in project textdb by TextDB.

the class TwitterSample method writeTwitterIndex.

public static void writeTwitterIndex() throws Exception {
    RelationManager relationManager = RelationManager.getRelationManager();
    relationManager.deleteTable(twitterClimateTable);
    relationManager.createTable(twitterClimateTable, "../index/twitter/", TwitterSchema.TWITTER_SCHEMA, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter dataWriter = relationManager.getTableDataWriter(twitterClimateTable);
    dataWriter.open();
    int counter = 0;
    JsonNode jsonNode = new ObjectMapper().readTree(new File(twitterFilePath));
    for (JsonNode tweet : jsonNode) {
        try {
            String text = tweet.get("text").asText();
            Long id = tweet.get("id").asLong();
            String tweetLink = "https://twitter.com/statuses/" + id;
            JsonNode userNode = tweet.get("user");
            String userScreenName = userNode.get("screen_name").asText();
            String userLink = "https://twitter.com/" + userScreenName;
            String userName = userNode.get("name").asText();
            String userDescription = userNode.get("description").asText();
            Integer userFollowersCount = userNode.get("followers_count").asInt();
            Integer userFriendsCount = userNode.get("friends_count").asInt();
            JsonNode geoTagNode = tweet.get("geo_tag");
            String state = geoTagNode.get("stateName").asText();
            String county = geoTagNode.get("countyName").asText();
            String city = geoTagNode.get("cityName").asText();
            String createAt = tweet.get("create_at").asText();
            Tuple tuple = new Tuple(TwitterSchema.TWITTER_SCHEMA, new TextField(text), new StringField(tweetLink), new StringField(userLink), new TextField(userScreenName), new TextField(userName), new TextField(userDescription), new IntegerField(userFollowersCount), new IntegerField(userFriendsCount), new TextField(state), new TextField(county), new TextField(city), new StringField(createAt));
            dataWriter.insertTuple(tuple);
            counter++;
        } catch (RuntimeException e) {
            e.printStackTrace();
            continue;
        }
    }
    dataWriter.close();
    System.out.println("write twitter data finished");
    System.out.println(counter + " tweets written");
}

Example 32 with DataWriter

use of edu.uci.ics.textdb.storage.DataWriter in project textdb by TextDB.

the class MedlineIndexWriter method writeMedlineIndex.

public static void writeMedlineIndex(Path medlineFilepath, String tableName) throws IOException, StorageException, ParseException {
    RelationManager relationManager = RelationManager.getRelationManager();
    DataWriter dataWriter = relationManager.getTableDataWriter(tableName);
    dataWriter.open();
    BufferedReader reader = Files.newBufferedReader(medlineFilepath);
    String line;
    while ((line = reader.readLine()) != null) {
        try {
            dataWriter.insertTuple(recordToTuple(line));
        } catch (Exception e) {
            e.printStackTrace();
        }
    }
    reader.close();
    dataWriter.close();
}

Example 33 with DataWriter

use of edu.uci.ics.textdb.storage.DataWriter in project textdb by TextDB.

the class OneToNBroadcastConnectorTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    RelationManager relationManager = RelationManager.getRelationManager();
    // create the people table and write tuples
    relationManager.createTable(PEOPLE_TABLE, "../index/test_tables/" + PEOPLE_TABLE, TestConstants.SCHEMA_PEOPLE, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter peopleDataWriter = relationManager.getTableDataWriter(PEOPLE_TABLE);
    peopleDataWriter.open();
    for (Tuple tuple : TestConstants.getSamplePeopleTuples()) {
        peopleDataWriter.insertTuple(tuple);
    }
    peopleDataWriter.close();
}

Example 34 with DataWriter

use of edu.uci.ics.textdb.storage.DataWriter in project textdb by TextDB.

the class ComparableMatcherTest method setUp.

@BeforeClass
public static void setUp() throws TextDBException {
    RelationManager relationManager = RelationManager.getRelationManager();
    // create the people table and write tuples
    relationManager.createTable(PEOPLE_TABLE, "../index/test_tables/" + PEOPLE_TABLE, TestConstants.SCHEMA_PEOPLE, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter peopleDataWriter = relationManager.getTableDataWriter(PEOPLE_TABLE);
    peopleDataWriter.open();
    for (Tuple tuple : TestConstants.getSamplePeopleTuples()) {
        peopleDataWriter.insertTuple(tuple);
    }
    peopleDataWriter.close();
}