Examples with WebPage org.apache.gora.examples.generated.WebPage used on opensource projects

Example 61 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class TestHBaseStore method assertTopLevelUnionsNull.

/**
   * Checks that when writing a top level union <code>['null','type']</code> 
   * with the option <code>RAW_ROOT_FIELDS_OPTION=true</code>
   * the column is not created, and when <code>RAW_ROOT_FIELDS_OPTION=false</code> 
   * the <code>null</code> value is serialized
   * with Avro.
   * @throws Exception
   */
@Test
public void assertTopLevelUnionsNull() throws Exception {
    WebPage page = webPageStore.newPersistent();
    // Write webpage data
    page.setUrl(new Utf8("http://example.com"));
    // This won't change internal field status to dirty, so
    page.setContent(null);
    // need to change it manually
    page.setDirty("content");
    webPageStore.put("com.example/http", page);
    webPageStore.flush();
    // Read directly from HBase
    Connection conn = ConnectionFactory.createConnection(conf);
    TableName webPageTab = TableName.valueOf("WebPage");
    Table table = conn.getTable(webPageTab);
    Get get = new Get(Bytes.toBytes("com.example/http"));
    org.apache.hadoop.hbase.client.Result result = table.get(get);
    table.close();
    byte[] contentBytes = result.getValue(Bytes.toBytes("content"), null);
    assertNull(webPageStore.get("com.example/http", new String[] { "content" }));
    assertTrue(contentBytes == null || contentBytes.length == 0);
}

Example 62 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DefaultFactoryTest method testCreateFilter_list_empty.

@Test
public void testCreateFilter_list_empty() throws Exception {
    FilterList<String, WebPage> filter = new FilterList<>();
    DBObject dbObject = filterFactory.createFilter(filter, store);
    assertEquals("{ }", dbObject.toString());
}

Example 63 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DefaultFactoryTest method testCreateFilter_mapField_notEquals.

@Test
public void testCreateFilter_mapField_notEquals() throws Exception {
    MapFieldValueFilter<String, WebPage> filter = createHeadersFilter();
    filter.setFilterOp(FilterOp.NOT_EQUALS);
    filter.setFilterIfMissing(true);
    DBObject dbObject = filterFactory.createFilter(filter, store);
    assertEquals("{ \"h.C·T\" : { \"$ne\" : \"text/html\"}}", dbObject.toString());
}

Example 64 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class TestMongoStore method addWebPage.

private void addWebPage() {
    String key = String.valueOf(keySequence++);
    WebPage p1 = webPageStore.newPersistent();
    p1.setUrl(new Utf8(key));
    p1.setHeaders(Collections.singletonMap((CharSequence) "header", (CharSequence) "value"));
    webPageStore.put(key, p1);
}

Example 65 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class TestSolrStore method testDeleteByQueryFields.

@Override
public void testDeleteByQueryFields() throws Exception {
    Query<String, WebPage> query;
    //test 5 - delete all with some fields
    WebPageDataCreator.createWebPageData(this.webPageStore);
    query = this.webPageStore.newQuery();
    query.setFields("outlinks", "parsedContent", "content");
    assertNumResults(this.webPageStore.newQuery(), URLS.length);
    this.webPageStore.deleteByQuery(query);
    this.webPageStore.flush();
    assertNumResults(this.webPageStore.newQuery(), URLS.length);
    //assert that data is deleted
    for (String SORTED_URL : SORTED_URLS) {
        WebPage page = this.webPageStore.get(SORTED_URL);
        assertNotNull(page);
        assertNotNull(page.getUrl());
        assertEquals(page.getUrl().toString(), SORTED_URL);
        assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery " + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
        assertEquals(0, page.getParsedContent().size());
        if (page.getContent() != null) {
            LOG.info("url: {}", page.getUrl().toString());
            LOG.info("limit: {}", page.getContent().limit());
        } else {
            assertNull(page.getContent());
        }
    }
    //test 6 - delete some with some fields
    WebPageDataCreator.createWebPageData(this.webPageStore);
    query = this.webPageStore.newQuery();
    query.setFields("url");
    String startKey = SORTED_URLS[NUM_KEYS];
    String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
    query.setStartKey(startKey);
    query.setEndKey(endKey);
    assertNumResults(this.webPageStore.newQuery(), URLS.length);
    this.webPageStore.deleteByQuery(query);
    this.webPageStore.flush();
    assertNumResults(query, 0);
}