Examples with TextField edu.uci.ics.texera.api.field.TextField used on opensource projects

Example 1 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class TwitterConverter method generateFieldsFromJson.

private List<IField> generateFieldsFromJson(String rawJsonData) {
    try {
        JsonNode tweet = new ObjectMapper().readTree(rawJsonData);
        String text = tweet.get("text").asText();
        Long id = tweet.get("id").asLong();
        String tweetLink = "https://twitter.com/statuses/" + id;
        JsonNode userNode = tweet.get("user");
        String userScreenName = userNode.get("screen_name").asText();
        String userLink = "https://twitter.com/" + userScreenName;
        String userName = userNode.get("name").asText();
        String userDescription = userNode.get("description").asText();
        Integer userFollowersCount = userNode.get("followers_count").asInt();
        Integer userFriendsCount = userNode.get("friends_count").asInt();
        JsonNode geoTagNode = tweet.get("geo_tag");
        String state = geoTagNode.get("stateName").asText();
        String county = geoTagNode.get("countyName").asText();
        String city = geoTagNode.get("cityName").asText();
        String createAt = tweet.get("create_at").asText();
        ZonedDateTime zonedCreateAt = ZonedDateTime.parse(createAt, DateTimeFormatter.ISO_INSTANT.withZone(ZoneId.systemDefault()));
        return Arrays.asList(new StringField(id.toString()), new TextField(text), new StringField(tweetLink), new StringField(userLink), new TextField(userScreenName), new TextField(userName), new TextField(userDescription), new IntegerField(userFollowersCount), new IntegerField(userFriendsCount), new TextField(state), new TextField(county), new TextField(city), new DateTimeField(zonedCreateAt.toLocalDateTime()));
    } catch (Exception e) {
        return Arrays.asList();
    }
}

Example 2 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class FileSourceOperatorTest method test4.

/*
     * Test FileSourceOperator with a Directory with recursive = true and maxDepth = 2.
     * 
     * The files under the recursive sub-directories with recursive depth 2 will be read.
     *     
     * expected results: test1.txt, test2.txt and test4.txt will be included
     */
@Test
public void test4() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName, true, 2);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)), new Tuple(schema, new TextField(tempFile4String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 3 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class FileSourceOperatorTest method test3.

/*
     * Test FileSourceOperator with a Directory with recursive = true and maxDepth = null.
     * 
     * All the files under the recursive sub-directories will be read.
     *     
     * expected results: test1.txt, test2.txt, test4.txt and test5.txt will be included
     */
@Test
public void test3() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName, true, null);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)), new Tuple(schema, new TextField(tempFile4String)), new Tuple(schema, new TextField(tempFile5String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 4 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class FileSourceOperatorTest method test2.

/*
     * Test FileSourceOperator with a Directory.
     * Optional parameters are all set to default. (only list files directly in this folder)
     * 
     * Only the files directly under this directory will be used.
     *     
     * expected results: test1.txt and test2.txt will be included.
     */
@Test
public void test2() throws Exception {
    String attrName = "content";
    Schema schema = new Schema(new Attribute(attrName, AttributeType.TEXT));
    FileSourcePredicate predicate = new FileSourcePredicate(tempFolderPath.toString(), attrName);
    FileSourceOperator fileSource = new FileSourceOperator(predicate);
    Tuple tuple;
    ArrayList<Tuple> exactResults = new ArrayList<>();
    fileSource.open();
    while ((tuple = fileSource.getNextTuple()) != null) {
        exactResults.add(tuple);
    }
    fileSource.close();
    List<Tuple> expectedResults = Arrays.asList(new Tuple(schema, new TextField(tempFile1String)), new Tuple(schema, new TextField(tempFile2String)));
    Assert.assertTrue(TestUtils.equals(expectedResults, exactResults));
}

Example 5 with TextField

use of edu.uci.ics.texera.api.field.TextField in project textdb by TextDB.

the class SpanTupleTest method testGetters.

@Test
public void testGetters() throws ParseException {
    // create data tuple first
    Attribute[] attributes = new Attribute[TestConstants.ATTRIBUTES_PEOPLE.length + 1];
    for (int count = 0; count < attributes.length - 1; count++) {
        attributes[count] = TestConstants.ATTRIBUTES_PEOPLE[count];
    }
    attributes[attributes.length - 1] = SchemaConstants.SPAN_LIST_ATTRIBUTE;
    List<IField> fields = new ArrayList<IField>(Arrays.asList(new IField[] { new StringField("bruce"), new StringField("lee"), new IntegerField(46), new DoubleField(5.50), new DateField(new SimpleDateFormat("MM-dd-yyyy").parse("01-14-1970")), new TextField("bruce was born in new york city and was grown up in los angeles") }));
    IField spanField = createSpanListField();
    fields.add(spanField);
    spanTuple = new Tuple(new Schema(attributes), fields.toArray(new IField[fields.size()]));
    IField spanFieldRetrieved = spanTuple.getField(SchemaConstants.SPAN_LIST);
    Assert.assertTrue(spanFieldRetrieved instanceof ListField);
    Assert.assertSame(spanField, spanFieldRetrieved);
}