Examples with WebPage org.apache.gora.examples.generated.WebPage used on opensource projects

Example 6 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class InfinispanStoreMapReduceTest method createWebPageDataStore.

@Override
protected DataStore<String, WebPage> createWebPageDataStore() throws IOException {
    conf = driver.getConfiguration();
    conf.set(ISPN_CONNECTION_STRING_KEY, driver.connectionString());
    try {
        InfinispanStore<String, WebPage> store = new InfinispanStore<>();
        store.setConf(conf);
        store.initialize(String.class, WebPage.class, new Properties());
        return store;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 7 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class InfinispanStoreTest method setUp.

@Before
public void setUp() throws Exception {
    GoraInfinispanTestDriver driver = getTestDriver();
    conf = driver.getConfiguration();
    conf.set(ISPN_CONNECTION_STRING_KEY, getTestDriver().connectionString());
    super.setUp();
    employeeDataStore = (InfinispanStore<String, Employee>) employeeStore;
    webPageDataStore = (InfinispanStore<String, WebPage>) webPageStore;
}

Example 8 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class TestHBaseStore method assertTopLevelUnions.

/**
   * Checks that when writing a top level union <code>['null','type']</code> 
   * the value is written in raw format
   * @throws Exception
   */
@Test
public void assertTopLevelUnions() throws Exception {
    WebPage page = webPageStore.newPersistent();
    // Write webpage data
    page.setUrl(new Utf8("http://example.com"));
    byte[] contentBytes = "example content in example.com".getBytes(Charset.defaultCharset());
    ByteBuffer buff = ByteBuffer.wrap(contentBytes);
    page.setContent(buff);
    webPageStore.put("com.example/http", page);
    webPageStore.flush();
    // Read directly from HBase
    HTable table = new HTable(conf, "WebPage");
    Get get = new Get(Bytes.toBytes("com.example/http"));
    org.apache.hadoop.hbase.client.Result result = table.get(get);
    byte[] bytesRead = result.getValue(Bytes.toBytes("content"), null);
    assertNotNull(bytesRead);
    assertTrue(Arrays.equals(bytesRead, contentBytes));
    table.close();
}

Example 9 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class JCacheGoraDataStoreTest method testDeleteByQueryFields.

@Test
public void testDeleteByQueryFields() throws Exception {
    DataStore<String, WebPage> store = super.webPageStore;
    BeanFactory<String, WebPage> beanFactory = new BeanFactoryImpl<>(String.class, WebPage.class);
    store.setBeanFactory(beanFactory);
    Query<String, WebPage> query;
    WebPageDataCreator.createWebPageData(store);
    query = store.newQuery();
    query.setFields("outlinks", "parsedContent", "content");
    Query<String, WebPage> newQuery = store.newQuery();
    newQuery.setStartKey(SORTED_URLS[0]);
    newQuery.setEndKey(SORTED_URLS[9]);
    newQuery.setFields("outlinks", "parsedContent", "content");
    DataStoreTestUtil.assertNumResults(newQuery, URLS.length);
    store.deleteByQuery(query);
    store.deleteByQuery(query);
    store.deleteByQuery(query);
    store.flush();
    DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length);
    for (String SORTED_URL : SORTED_URLS) {
        WebPage page = store.get(SORTED_URL);
        assertNotNull(page);
        assertNotNull(page.getUrl());
        assertEquals(page.getUrl().toString(), SORTED_URL);
        assertEquals("Map of Outlinks should have a size of '0' as the deleteByQuery " + "not only removes the data but also the data structure.", 0, page.getOutlinks().size());
        assertEquals(0, page.getParsedContent().size());
        if (page.getContent() != null) {
            LOG.info("url:" + page.getUrl().toString());
            LOG.info("limit:" + page.getContent().limit());
        } else {
            assertNull(page.getContent());
        }
    }
    WebPageDataCreator.createWebPageData(store);
    query = store.newQuery();
    query.setFields("url");
    String startKey = SORTED_URLS[NUM_KEYS];
    String endKey = SORTED_URLS[SORTED_URLS.length - NUM_KEYS];
    query.setStartKey(startKey);
    query.setEndKey(endKey);
    DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length);
    store.deleteByQuery(query);
    store.deleteByQuery(query);
    store.deleteByQuery(query);
    store.flush();
    DataStoreTestUtil.assertNumResults(store.newQuery(), URLS.length);
    for (int i = 0; i < URLS.length; i++) {
        WebPage page = store.get(URLS[i]);
        assertNotNull(page);
        if (URLS[i].compareTo(startKey) < 0 || URLS[i].compareTo(endKey) > 0) {
            DataStoreTestUtil.assertWebPage(page, i);
        } else {
            assertNull(page.getUrl());
            assertNotNull(page.getOutlinks());
            assertNotNull(page.getParsedContent());
            assertNotNull(page.getContent());
            assertTrue(page.getOutlinks().size() > 0);
            assertTrue(page.getParsedContent().size() > 0);
        }
    }
}

Example 10 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class JCacheGoraDataStoreTest method testGetWithFields.

@Test
public void testGetWithFields() throws Exception {
    DataStore<String, WebPage> store = super.webPageStore;
    BeanFactory<String, WebPage> beanFactory = new BeanFactoryImpl<>(String.class, WebPage.class);
    store.setBeanFactory(beanFactory);
    WebPageDataCreator.createWebPageData(store);
    String[] interestFields = new String[2];
    interestFields[0] = "url";
    interestFields[1] = "content";
    WebPage page = store.get(URLS[1], interestFields);
    assertNotNull(page);
    assertNotNull(page.getUrl());
    assertEquals(page.getUrl().toString(), URLS[1]);
    assertNotNull(page.getContent());
    assertEquals("Map of Outlinks should have a size of '0' as it is omitted at retrieval", 0, page.getOutlinks().size());
    assertEquals("Map of Parsed Content should have a size of '0' as it is omitted at retrieval", 0, page.getParsedContent().size());
}