Examples with WebPage org.apache.gora.examples.generated.WebPage used on opensource projects

Example 26 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testUpdateWebPageRemoveField.

public static void testUpdateWebPageRemoveField(DataStore<String, WebPage> dataStore) throws Exception {
    dataStore.createSchema();
    String[] urls = { "http://a.com/a", "http://b.com/b", "http://c.com/c", "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
    String header = "header";
    String[] headers = { "firstHeader", "secondHeader", "thirdHeader", "fourthHeader", "fifthHeader", "sixthHeader" };
    for (String url : urls) {
        WebPage webPage = WebPage.newBuilder().build();
        webPage.setUrl(new Utf8(url));
        webPage.setHeaders(new HashMap<CharSequence, CharSequence>());
        for (int j = 0; j < headers.length; j++) {
            webPage.getHeaders().put(new Utf8(header + j), new Utf8(headers[j]));
        }
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    // nullable map field removal test
    for (String url : urls) {
        WebPage webPage = dataStore.get(url);
        webPage.setHeaders(null);
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    for (String url : urls) {
        WebPage webPage = dataStore.get(url);
        assertNull(webPage.getHeaders());
    }
}

Example 27 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testDeleteByQueryFields.

public static void testDeleteByQueryFields(DataStore<String, WebPage> store) throws Exception {
    Query<String, WebPage> query;
    //test 5 - delete all with some fields
    WebPageDataCreator.createWebPageData(store);
    query = store.newQuery();
    query.setFields("outlinks", "parsedContent", "content");
    assertNumResults(store.newQuery(), URLS.length);
    store.deleteByQuery(query);
    store.flush();
    assertNumResults(store.newQuery(), URLS.length);
    //assert that data is deleted
    for (String SORTED_URL : SORTED_URLS) {
        WebPage page = store.get(SORTED_URL);
        assertNotNull(page);
        assertNotNull(page.getUrl());
        assertEquals(page.getUrl().toString(), SORTED_URL);
        assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery " + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
        assertEquals(0, page.getParsedContent().size());
        if (page.getContent() != null) {
            LOG.info("url:" + page.getUrl().toString());
            LOG.info("limit:" + page.getContent().limit());
        } else {
            assertNull(page.getContent());
        }
    }
    //test 6 - delete some with some fields
    WebPageDataCreator.createWebPageData(store);
    query = store.newQuery();
    query.setFields("url");
    String startKey = SORTED_URLS[NUM_KEYS];
    String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
    query.setStartKey(startKey);
    query.setEndKey(endKey);
    assertNumResults(store.newQuery(), URLS.length);
    store.deleteByQuery(query);
    store.flush();
    assertNumResults(store.newQuery(), URLS.length);
    //assert that data is deleted
    for (int i = 0; i < URLS.length; i++) {
        WebPage page = store.get(URLS[i]);
        assertNotNull(page);
        if (URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) > 0) {
            //not deleted
            assertWebPage(page, i);
        } else {
            //deleted
            assertNull(page.getUrl());
            assertNotNull(page.getOutlinks());
            assertNotNull(page.getParsedContent());
            assertNotNull(page.getContent());
            assertTrue(page.getOutlinks().size() > 0);
            assertTrue(page.getParsedContent().size() > 0);
        }
    }
}

Example 28 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testPutBytes.

public static byte[] testPutBytes(DataStore<String, WebPage> store) throws Exception {
    store.createSchema();
    WebPage page = WebPage.newBuilder().build();
    page.setUrl(new Utf8("http://example.com"));
    byte[] contentBytes = "example content in example.com".getBytes(Charset.defaultCharset());
    ByteBuffer buff = ByteBuffer.wrap(contentBytes);
    page.setContent(buff);
    store.put("com.example/http", page);
    store.close();
    return contentBytes;
}

Example 29 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DefaultFactoryTest method testCreateFilter_singleField_notEquals.

@Test
public void testCreateFilter_singleField_notEquals() throws Exception {
    SingleFieldValueFilter<String, WebPage> filter = createUrlFilter();
    filter.setFilterOp(FilterOp.NOT_EQUALS);
    filter.setFilterIfMissing(true);
    DBObject dbObject = filterFactory.createFilter(filter, store);
    assertEquals("{ \"url\" : { \"$ne\" : \"http://www.example.com\"}}", dbObject.toString());
}

Example 30 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DefaultFactoryTest method testCreateFilter_singleField_equalsOrNull.

@Test
public void testCreateFilter_singleField_equalsOrNull() throws Exception {
    SingleFieldValueFilter<String, WebPage> filter = createUrlFilter();
    filter.setFilterOp(FilterOp.EQUALS);
    // include doc with missing field
    filter.setFilterIfMissing(false);
    DBObject dbObject = filterFactory.createFilter(filter, store);
    assertEquals("{ \"$or\" : [ { \"url\" : { \"$exists\" : false}} , { \"url\" : \"http://www.example.com\"}]}", dbObject.toString());
}