Examples with DataWriter edu.uci.ics.texera.storage.DataWriter used on opensource projects

Example 6 with DataWriter

use of edu.uci.ics.texera.storage.DataWriter in project textdb by TextDB.

the class TwitterSample method createTwitterTable.

/**
 * A helper function to create a table and write twitter data into it.
 *
 * @param tableName
 * @param twitterJsonSourceOperator, a source operator that provides the input raw twitter JSON string tuples
 * @return
 */
public static int createTwitterTable(String tableName, ISourceOperator twitterJsonSourceOperator) {
    TwitterJsonConverter twitterJsonConverter = new TwitterJsonConverterPredicate("twitterJson").newOperator();
    TupleSink tupleSink = new TupleSinkPredicate(null, null).newOperator();
    twitterJsonConverter.setInputOperator(twitterJsonSourceOperator);
    tupleSink.setInputOperator(twitterJsonConverter);
    // open the workflow plan and get the output schema
    tupleSink.open();
    // create the table with TupleSink's output schema
    RelationManager relationManager = RelationManager.getInstance();
    if (relationManager.checkTableExistence(tableName)) {
        relationManager.deleteTable(tableName);
    }
    relationManager.createTable(tableName, Utils.getDefaultIndexDirectory().resolve(tableName), tupleSink.getOutputSchema(), LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter dataWriter = relationManager.getTableDataWriter(tableName);
    dataWriter.open();
    Tuple tuple;
    int counter = 0;
    while ((tuple = tupleSink.getNextTuple()) != null) {
        dataWriter.insertTuple(tuple);
        counter++;
    }
    dataWriter.close();
    tupleSink.close();
    return counter;
}

Example 7 with DataWriter

use of edu.uci.ics.texera.storage.DataWriter in project textdb by TextDB.

the class RegexMatcherTestHelper method writeTestTables.

public static void writeTestTables() throws TexeraException {
    RelationManager relationManager = RelationManager.getInstance();
    // create the people table and write tuples
    relationManager.createTable(PEOPLE_TABLE, TestUtils.getDefaultTestIndex().resolve(PEOPLE_TABLE), TestConstants.SCHEMA_PEOPLE, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter peopleDataWriter = relationManager.getTableDataWriter(PEOPLE_TABLE);
    peopleDataWriter.open();
    for (Tuple tuple : TestConstants.getSamplePeopleTuples()) {
        peopleDataWriter.insertTuple(tuple);
    }
    peopleDataWriter.close();
    // create the corporation table and write tuples
    relationManager.createTable(CORP_TABLE, TestUtils.getDefaultTestIndex().resolve(CORP_TABLE), RegexTestConstantsCorp.SCHEMA_CORP, LuceneAnalyzerConstants.nGramAnalyzerString(3));
    DataWriter corpDataWriter = relationManager.getTableDataWriter(CORP_TABLE);
    corpDataWriter.open();
    for (Tuple tuple : RegexTestConstantsCorp.getSampleCorpTuples()) {
        corpDataWriter.insertTuple(tuple);
    }
    corpDataWriter.close();
    // create the staff table
    relationManager.createTable(STAFF_TABLE, TestUtils.getDefaultTestIndex().resolve(STAFF_TABLE), RegexTestConstantStaff.SCHEMA_STAFF, LuceneAnalyzerConstants.nGramAnalyzerString(3));
    DataWriter staffDataWriter = relationManager.getTableDataWriter(STAFF_TABLE);
    staffDataWriter.open();
    for (Tuple tuple : RegexTestConstantStaff.getSampleStaffTuples()) {
        staffDataWriter.insertTuple(tuple);
    }
    staffDataWriter.close();
    // create the text table
    relationManager.createTable(TEXT_TABLE, TestUtils.getDefaultTestIndex().resolve(TEXT_TABLE), RegexTestConstantsText.SCHEMA_TEXT, LuceneAnalyzerConstants.nGramAnalyzerString(3));
    DataWriter textDataWriter = relationManager.getTableDataWriter(TEXT_TABLE);
    textDataWriter.open();
    for (Tuple tuple : RegexTestConstantsText.getSampleTextTuples()) {
        textDataWriter.insertTuple(tuple);
    }
    textDataWriter.close();
}

Example 8 with DataWriter

use of edu.uci.ics.texera.storage.DataWriter in project textdb by TextDB.

the class OneToNBroadcastConnectorTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    RelationManager relationManager = RelationManager.getInstance();
    // create the people table and write tuples
    relationManager.createTable(PEOPLE_TABLE, TestUtils.getDefaultTestIndex().resolve(PEOPLE_TABLE), TestConstants.SCHEMA_PEOPLE, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter peopleDataWriter = relationManager.getTableDataWriter(PEOPLE_TABLE);
    peopleDataWriter.open();
    for (Tuple tuple : TestConstants.getSamplePeopleTuples()) {
        peopleDataWriter.insertTuple(tuple);
    }
    peopleDataWriter.close();
}

Example 9 with DataWriter

use of edu.uci.ics.texera.storage.DataWriter in project textdb by TextDB.

the class JoinTestHelper method insertToTable.

public static void insertToTable(String tableName, Tuple... tuples) throws StorageException {
    RelationManager relationManager = RelationManager.getInstance();
    DataWriter tableDataWriter = relationManager.getTableDataWriter(tableName);
    tableDataWriter.open();
    for (Tuple tuple : Arrays.asList(tuples)) {
        tableDataWriter.insertTuple(tuple);
    }
    tableDataWriter.close();
}

Example 10 with DataWriter

use of edu.uci.ics.texera.storage.DataWriter in project textdb by TextDB.

the class DictionaryMatcherTestHelper method writeTestTables.

public static void writeTestTables() throws TexeraException {
    RelationManager relationManager = RelationManager.getInstance();
    // create the people table and write tuples
    relationManager.createTable(PEOPLE_TABLE, TestUtils.getDefaultTestIndex().resolve(PEOPLE_TABLE), TestConstants.SCHEMA_PEOPLE, LuceneAnalyzerConstants.standardAnalyzerString());
    DataWriter peopleDataWriter = relationManager.getTableDataWriter(PEOPLE_TABLE);
    peopleDataWriter.open();
    for (Tuple tuple : TestConstants.getSamplePeopleTuples()) {
        peopleDataWriter.insertTuple(tuple);
    }
    peopleDataWriter.close();
    // create the people table and write tuples in Chinese
    relationManager.createTable(CHINESE_TABLE, TestUtils.getDefaultTestIndex().resolve(CHINESE_TABLE), TestConstantsChinese.SCHEMA_PEOPLE, LuceneAnalyzerConstants.chineseAnalyzerString());
    DataWriter chineseDataWriter = relationManager.getTableDataWriter(CHINESE_TABLE);
    chineseDataWriter.open();
    for (Tuple tuple : TestConstantsChinese.getSamplePeopleTuples()) {
        chineseDataWriter.insertTuple(tuple);
    }
    chineseDataWriter.close();
}