Examples with TweetParser org.apache.asterix.external.parser.TweetParser used on opensource projects

Example 1 with TweetParser

use of org.apache.asterix.external.parser.TweetParser in project asterixdb by apache.

the class TweetParserTest method closedRecordTypeTest.

@Test
public void closedRecordTypeTest() throws IOException, URISyntaxException {
    // contruct type
    IAType geoFieldType = new ARecordType("GeoType", new String[] { "coordinates" }, new IAType[] { new AOrderedListType(AFLOAT, "point") }, true);
    ARecordType tweetRecordType = new ARecordType("TweetType", new String[] { "id", "geo" }, new IAType[] { AINT64, geoFieldType }, true);
    TweetParser parser = new TweetParser(tweetRecordType);
    List<String> lines = Files.readAllLines(Paths.get(getClass().getResource("/test_tweets.txt").toURI()));
    ByteArrayOutputStream is = new ByteArrayOutputStream();
    DataOutput output = new DataOutputStream(is);
    int regularCount = 0;
    for (int iter1 = 0; iter1 < lines.size(); iter1++) {
        GenericRecord<String> record = new GenericRecord<>();
        record.set(lines.get(iter1));
        try {
            parser.parse(record, output);
            regularCount++;
        } catch (HyracksDataException e) {
            Assert.assertTrue(e.toString().contains("Non-null") && (iter1 == 0 || iter1 == 1));
        }
    }
    Assert.assertTrue(regularCount == 4);
}

Example 2 with TweetParser

use of org.apache.asterix.external.parser.TweetParser in project asterixdb by apache.

the class TweetParserTest method openRecordTypeTest.

@Test
public void openRecordTypeTest() throws IOException, URISyntaxException {
    String[] ids = { "720549057849114629", "668950503552864257", "668945640186101761", "263602997047730177", "668948268605403136", "741701526859567104" };
    // contruct type
    IAType geoFieldType = new ARecordType("GeoType", new String[] { "coordinates" }, new IAType[] { new AOrderedListType(AFLOAT, "point") }, true);
    List<IAType> unionTypeList = new ArrayList<>();
    unionTypeList.add(geoFieldType);
    unionTypeList.add(ANULL);
    unionTypeList.add(AMISSING);
    IAType geoUnionType = new AUnionType(unionTypeList, "GeoType?");
    ARecordType tweetRecordType = new ARecordType("TweetType", new String[] { "id", "geo" }, new IAType[] { AINT64, geoUnionType }, true);
    TweetParser parser = new TweetParser(tweetRecordType);
    List<String> lines = Files.readAllLines(Paths.get(getClass().getResource("/test_tweets.txt").toURI()));
    ByteArrayOutputStream is = new ByteArrayOutputStream();
    DataOutput output = new DataOutputStream(is);
    for (int iter1 = 0; iter1 < lines.size(); iter1++) {
        GenericRecord<String> record = new GenericRecord<>();
        record.set(lines.get(iter1));
        try {
            parser.parse(record, output);
        } catch (HyracksDataException e) {
            e.printStackTrace();
            Assert.fail("Unexpected failure in parser.");
        }
        Assert.assertTrue((PA.getValue(parser, "aInt64")).toString().equals(ids[iter1]));
    }
}