Examples with WebPage org.apache.gora.examples.generated.WebPage used on opensource projects

Example 36 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testUpdateWebPagePutToNotNullableMap.

public static void testUpdateWebPagePutToNotNullableMap(DataStore<String, WebPage> dataStore) throws Exception {
    dataStore.createSchema();
    String[] urls = { "http://a.com/a", "http://b.com/b", "http://c.com/c", "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
    String anchor = "anchor";
    // putting evens
    for (String url : urls) {
        WebPage webPage = WebPage.newBuilder().build();
        webPage.setUrl(new Utf8(url));
        for (int j = 0; j < urls.length; j += 2) {
            webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
        }
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    // putting odds
    for (String url : urls) {
        WebPage webPage = dataStore.get(url);
        webPage.getOutlinks().clear();
        for (int j = 1; j < urls.length; j += 2) {
            webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
        }
        // test for double put of same entries
        for (int j = 1; j < urls.length; j += 2) {
            webPage.getOutlinks().put(new Utf8(anchor + j), new Utf8(urls[j]));
        }
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    for (String url : urls) {
        WebPage webPage = dataStore.get(url);
        int count = 0;
        for (int j = 1; j < urls.length; j += 2) {
            CharSequence link = webPage.getOutlinks().get(new Utf8(anchor + j));
            assertNotNull(link);
            assertEquals(urls[j], link.toString());
            count++;
        }
        assertEquals(count, webPage.getOutlinks().size());
    }
}

Example 37 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testPutNested.

public static void testPutNested(DataStore<String, WebPage> store) throws Exception {
    String revUrl = "foo.com:http/";
    String url = "http://foo.com/";
    store.createSchema();
    WebPage page = WebPage.newBuilder().build();
    Metadata metadata = Metadata.newBuilder().build();
    metadata.setVersion(1);
    metadata.getData().put(new Utf8("foo"), new Utf8("baz"));
    page.setMetadata(metadata);
    page.setUrl(new Utf8(url));
    store.put(revUrl, page);
    store.flush();
    page = store.get(revUrl);
    metadata = page.getMetadata();
    assertNotNull(metadata);
    assertEquals(1, metadata.getVersion().intValue());
    assertEquals(new Utf8("baz"), metadata.getData().get(new Utf8("foo")));
}

Example 38 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testGetWebPage.

private static void testGetWebPage(DataStore<String, WebPage> store, String[] fields) throws Exception {
    createWebPageData(store);
    for (int i = 0; i < URLS.length; i++) {
        WebPage page = store.get(URLS[i], fields);
        assertWebPage(page, i);
    }
}

Example 39 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testUpdateWebPagePutToArray.

/**
   * Here we create 7 {@link org.apache.gora.examples.generated.WebPage}
   * objects and populate field data before flushing the objects to the 
   * datastore. We then get the objects, adding data to the 'content' and
   * 'parsedContent' fields before clearing the 'outlinks' field and 
   * re-populating it. This data is then flushed to the datastore. 
   * Finally we get the {@link org.apache.gora.examples.generated.WebPage}
   * objects and make various assertions over verious fields. This tests 
   * that we can update fields and that data can be written and read correctly.
   * @param dataStore
   * @throws IOException
   * @throws Exception
   */
public static void testUpdateWebPagePutToArray(DataStore<String, WebPage> dataStore) throws Exception {
    dataStore.createSchema();
    String[] urls = { "http://a.com/a", "http://b.com/b", "http://c.com/c", "http://d.com/d", "http://e.com/e", "http://f.com/f", "http://g.com/g" };
    String content = "content";
    String parsedContent = "parsedContent";
    int parsedContentCount = 0;
    for (int i = 0; i < urls.length; i++) {
        WebPage webPage = WebPage.newBuilder().build();
        webPage.setUrl(new Utf8(urls[i]));
        for (parsedContentCount = 0; parsedContentCount < 5; parsedContentCount++) {
            webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
        }
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    for (int i = 0; i < urls.length; i++) {
        WebPage webPage = dataStore.get(urls[i]);
        webPage.setContent(ByteBuffer.wrap(ByteUtils.toBytes(content + i)));
        for (parsedContentCount = 5; parsedContentCount < 10; parsedContentCount++) {
            webPage.getParsedContent().add(new Utf8(parsedContent + i + "," + parsedContentCount));
        }
        dataStore.put(webPage.getUrl().toString(), webPage);
    }
    dataStore.flush();
    for (int i = 0; i < urls.length; i++) {
        WebPage webPage = dataStore.get(urls[i]);
        assertEquals(content + i, ByteUtils.toString(toByteArray(webPage.getContent())));
        assertEquals(10, webPage.getParsedContent().size());
        int j = 0;
        for (CharSequence pc : webPage.getParsedContent()) {
            assertEquals(parsedContent + i + "," + j, pc.toString());
            j++;
        }
    }
}

Example 40 with WebPage

use of org.apache.gora.examples.generated.WebPage in project gora by apache.

the class DataStoreTestUtil method testGetEmployeeWithFields.

public static void testGetEmployeeWithFields(DataStore<String, Employee> dataStore) throws Exception {
    Employee employee = DataStoreTestUtil.createEmployee();
    WebPage webpage = createWebPage();
    employee.setWebpage(webpage);
    Employee boss = createBoss();
    employee.setBoss(boss);
    String ssn = employee.getSsn().toString();
    dataStore.put(ssn, employee);
    dataStore.flush();
    String[] fields = AvroUtils.getPersistentFieldNames(employee);
    for (Set<String> subset : StringUtils.powerset(fields)) {
        if (subset.isEmpty())
            continue;
        Employee after = dataStore.get(ssn, subset.toArray(new String[subset.size()]));
        Employee expected = Employee.newBuilder().build();
        for (String field : subset) {
            int index = expected.getSchema().getField(field).pos();
            expected.put(index, employee.get(index));
        }
        assertEqualEmployeeObjects(expected, after);
    }
}