use of biblemulticonverter.schema.usx3.Row in project beam by apache.
the class BigtableServiceImplTest method testRead.
/**
* This test ensures that protobuf creation and interactions with {@link BigtableDataClient} work
* as expected.
*
* @throws IOException
* @throws InterruptedException
*/
@Test
public void testRead() throws IOException {
ByteKey start = ByteKey.copyFrom("a".getBytes(StandardCharsets.UTF_8));
ByteKey end = ByteKey.copyFrom("b".getBytes(StandardCharsets.UTF_8));
when(mockBigtableSource.getRanges()).thenReturn(Arrays.asList(ByteKeyRange.of(start, end)));
when(mockBigtableSource.getTableId()).thenReturn(StaticValueProvider.of(TABLE_ID));
@SuppressWarnings("unchecked") ResultScanner<Row> mockResultScanner = Mockito.mock(ResultScanner.class);
Row expectedRow = Row.newBuilder().setKey(ByteString.copyFromUtf8("a")).build();
when(mockResultScanner.next()).thenReturn(expectedRow).thenReturn(null);
when(mockBigtableDataClient.readRows(any(ReadRowsRequest.class))).thenReturn(mockResultScanner);
BigtableService.Reader underTest = new BigtableServiceImpl.BigtableReaderImpl(mockSession, mockBigtableSource);
underTest.start();
Assert.assertEquals(expectedRow, underTest.getCurrentRow());
Assert.assertFalse(underTest.advance());
underTest.close();
verify(mockResultScanner, times(1)).close();
verifyMetricWasSet("google.bigtable.v2.ReadRows", "ok", 1);
}
use of biblemulticonverter.schema.usx3.Row in project YCSB by brianfrankcooper.
the class GoogleBigtableClient method scan.
@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
setTable(table);
RowFilter filter = RowFilter.newBuilder().setFamilyNameRegexFilterBytes(ByteStringer.wrap(columnFamilyBytes)).build();
if (fields != null && fields.size() > 0) {
Builder filterChain = RowFilter.Chain.newBuilder();
filterChain.addFilters(filter);
filterChain.addFilters(RowFilter.newBuilder().setCellsPerColumnLimitFilter(1).build());
int count = 0;
// usually "field#" so pre-alloc
final StringBuilder regex = new StringBuilder(fields.size() * 6);
for (final String field : fields) {
if (count++ > 0) {
regex.append("|");
}
regex.append(field);
}
filterChain.addFilters(RowFilter.newBuilder().setColumnQualifierRegexFilter(ByteStringer.wrap(regex.toString().getBytes()))).build();
filter = RowFilter.newBuilder().setChain(filterChain.build()).build();
}
final RowRange range = RowRange.newBuilder().setStartKeyClosed(ByteStringer.wrap(startkey.getBytes())).build();
final RowSet rowSet = RowSet.newBuilder().addRowRanges(range).build();
final ReadRowsRequest.Builder rrr = ReadRowsRequest.newBuilder().setTableNameBytes(ByteStringer.wrap(lastTableBytes)).setFilter(filter).setRows(rowSet);
List<Row> rows;
try {
rows = client.readRowsAsync(rrr.build()).get();
if (rows == null || rows.isEmpty()) {
return Status.NOT_FOUND;
}
int numResults = 0;
for (final Row row : rows) {
final HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>(fields != null ? fields.size() : 10);
for (final Family family : row.getFamiliesList()) {
if (Arrays.equals(family.getNameBytes().toByteArray(), columnFamilyBytes)) {
for (final Column column : family.getColumnsList()) {
// we should only have a single cell per column
rowResult.put(column.getQualifier().toString(UTF8_CHARSET), new ByteArrayByteIterator(column.getCells(0).getValue().toByteArray()));
if (debug) {
System.out.println("Result for field: " + column.getQualifier().toString(UTF8_CHARSET) + " is: " + column.getCells(0).getValue().toString(UTF8_CHARSET));
}
}
}
}
result.add(rowResult);
numResults++;
if (numResults >= recordcount) {
// if hit recordcount, bail out
break;
}
}
return Status.OK;
} catch (InterruptedException e) {
System.err.println("Interrupted during scan: " + e);
Thread.currentThread().interrupt();
return Status.ERROR;
} catch (ExecutionException e) {
System.err.println("Exception during scan: " + e);
return Status.ERROR;
}
}
use of biblemulticonverter.schema.usx3.Row in project BibleMultiConverter by schierlm.
the class USX3 method doImportBook.
@Override
protected ParatextBook doImportBook(File inputFile) throws Exception {
if (!inputFile.getName().toLowerCase().endsWith(".usx"))
return null;
ValidateXML.validateFileBeforeParsing(getSchema(), inputFile);
JAXBContext ctx = JAXBContext.newInstance(ObjectFactory.class.getPackage().getName());
XMLInputFactory xif = XMLInputFactory.newFactory();
XMLStreamReader xsr = xif.createXMLStreamReader(new FileInputStream(inputFile));
Unmarshaller u = ctx.createUnmarshaller();
u.setListener(unmarshallerLocationListener);
unmarshallerLocationListener.setXMLStreamReader(inputFile.getName(), xsr);
Usx doc = (Usx) u.unmarshal(xsr);
xsr.close();
ParatextBook.ParatextID id = ParatextBook.ParatextID.fromIdentifier(doc.getBook().getCode().toUpperCase());
if (id == null) {
System.out.println("WARNING: Skipping book with unknown ID: " + doc.getBook().getCode());
return null;
}
ParatextBook result = new ParatextBook(id, doc.getBook().getContent());
ParatextCharacterContent charContent = null;
for (Object o : doc.getParaOrTableOrChapter()) {
if (o instanceof Para) {
Para para = (Para) o;
if (BOOK_HEADER_ATTRIBUTE_TAGS.contains(para.getStyle().value())) {
String value = "";
for (Object oo : para.getContent()) {
if (oo instanceof String) {
value += ((String) oo).replaceAll("[ \r\n\t]+", " ");
} else {
throw new RuntimeException("Unsupported content in attribute: " + oo.getClass());
}
}
result.getAttributes().put(para.getStyle().value(), value);
charContent = null;
} else if (para.getStyle() == ParaStyle.PB) {
if (charContent == null) {
charContent = new ParatextCharacterContent();
result.getContent().add(charContent);
}
charContent.getContent().add(new ParatextCharacterContent.AutoClosingFormatting(ParatextCharacterContent.AutoClosingFormattingKind.PAGE_BREAK, false));
} else if (PARA_STYLE_UNSUPPORTED.contains(para.getStyle())) {
// skip
charContent = null;
} else {
result.getContent().add(new ParatextBook.ParagraphStart(PARA_STYLE_MAP.get(para.getStyle())));
charContent = null;
if (!para.getContent().isEmpty()) {
charContent = new ParatextCharacterContent();
result.getContent().add(charContent);
parseCharContent(para.getContent(), charContent);
}
}
} else if (o instanceof Table) {
Table table = (Table) o;
for (Row row : table.getRow()) {
result.getContent().add(new ParatextBook.ParagraphStart(ParatextBook.ParagraphKind.TABLE_ROW));
for (Object oo : row.getVerseOrCell()) {
if (oo instanceof Verse) {
Verse verse = (Verse) oo;
ParatextCharacterContent.ParatextCharacterContentPart verseStartOrEnd = handleVerse(verse);
charContent = new ParatextCharacterContent();
result.getContent().add(charContent);
charContent.getContent().add(verseStartOrEnd);
} else if (oo instanceof Cell) {
Cell cell = (Cell) oo;
result.getContent().add(new ParatextBook.TableCellStart(cell.getStyle().value()));
charContent = new ParatextCharacterContent();
result.getContent().add(charContent);
parseCharContent(cell.getContent(), charContent);
} else {
throw new IOException("Unsupported table row element: " + o.getClass().getName());
}
}
}
charContent = null;
} else if (o instanceof Chapter) {
Chapter chapter = (Chapter) o;
if (chapter.getSid() != null) {
// Assume start chapter
result.getContent().add(new ParatextBook.ChapterStart(new ChapterIdentifier(result.getId(), ((Chapter) o).getNumber().intValue())));
} else if (chapter.getEid() != null) {
// Assume end chapter
ChapterIdentifier location = ChapterIdentifier.fromLocationString(chapter.getEid());
if (location == null) {
throw new IOException("Invalid chapter eid found: " + chapter.getEid());
}
result.getContent().add(new ParatextBook.ChapterEnd(location));
} else {
throw new IOException("Invalid chapter found, both sid and eid are undefined: " + chapter);
}
charContent = null;
} else if (o instanceof Note) {
if (charContent == null) {
charContent = new ParatextCharacterContent();
result.getContent().add(charContent);
}
Note note = (Note) o;
ParatextCharacterContent.FootnoteXref nx = new ParatextCharacterContent.FootnoteXref(NOTE_STYLE_MAP.get(note.getStyle()), note.getCaller());
charContent.getContent().add(nx);
parseCharContent(note.getContent(), nx);
} else if (o instanceof Sidebar) {
System.out.println("WARNING: Skipping sidebar (study bible content)");
charContent = null;
} else {
throw new IOException("Unsupported book level element: " + o.getClass().getName());
}
}
return result;
}
use of biblemulticonverter.schema.usx3.Row in project BibleMultiConverter by schierlm.
the class USX3 method doExportBook.
@Override
protected void doExportBook(ParatextBook book, File outFile) throws Exception {
ObjectFactory of = new ObjectFactory();
Usx usx = of.createUsx();
usx.setVersion("3.0");
usx.setBook(of.createBook());
usx.getBook().setStyle("id");
usx.getBook().setCode(book.getId().getIdentifier());
usx.getBook().setContent(book.getBibleName());
for (Map.Entry<String, String> attr : book.getAttributes().entrySet()) {
Para para = new Para();
para.setStyle(ParaStyle.fromValue(attr.getKey()));
para.getContent().add(attr.getValue());
usx.getParaOrTableOrChapter().add(para);
}
book.accept(new ParatextBook.ParatextBookContentVisitor<IOException>() {
List<Object> currentContent = null;
Table currentTable = null;
@Override
public void visitChapterStart(ChapterIdentifier location) throws IOException {
Chapter ch = new Chapter();
ch.setStyle("c");
ch.setSid(location.toString());
ch.setNumber(BigInteger.valueOf(location.chapter));
usx.getParaOrTableOrChapter().add(ch);
currentContent = null;
currentTable = null;
}
@Override
public void visitChapterEnd(ChapterIdentifier location) throws IOException {
Chapter ch = new Chapter();
ch.setEid(location.toString());
usx.getParaOrTableOrChapter().add(ch);
currentContent = null;
currentTable = null;
}
@Override
public void visitParagraphStart(ParatextBook.ParagraphKind kind) throws IOException {
if (kind == ParatextBook.ParagraphKind.TABLE_ROW) {
if (currentTable == null) {
currentTable = new Table();
usx.getParaOrTableOrChapter().add(currentTable);
}
Row row = new Row();
row.setStyle("tr");
currentTable.getRow().add(row);
currentContent = currentTable.getRow().get(currentTable.getRow().size() - 1).getVerseOrCell();
} else {
Para para = new Para();
para.setStyle(PARA_KIND_MAP.get(kind));
usx.getParaOrTableOrChapter().add(para);
currentContent = para.getContent();
currentTable = null;
}
}
@Override
public void visitTableCellStart(String tag) throws IOException {
if (currentTable == null) {
System.out.println("WARNING: Table cell outside of table");
return;
}
Row currentRow = currentTable.getRow().get(currentTable.getRow().size() - 1);
Cell cell = new Cell();
cell.setAlign(tag.contains("r") ? CellAlign.END : CellAlign.START);
cell.setStyle(CellStyle.fromValue(tag));
currentRow.getVerseOrCell().add(cell);
currentContent = cell.getContent();
}
@Override
public void visitParatextCharacterContent(ParatextCharacterContent content) throws IOException {
if (currentContent == null)
visitParagraphStart(ParatextBook.ParagraphKind.PARAGRAPH_P);
content.accept(new USX3.USXCharacterContentVisitor(currentContent));
}
});
JAXBContext ctx = JAXBContext.newInstance(ObjectFactory.class.getPackage().getName());
Marshaller m = ctx.createMarshaller();
if (!Boolean.getBoolean("biblemulticonverter.skipxmlvalidation"))
m.setSchema(getSchema());
m.marshal(usx, new UnifiedScriptureXMLWriter(new FileWriter(outFile), "UTF-8"));
}
use of biblemulticonverter.schema.usx3.Row in project simple-bigtable by spotify.
the class FamiliesReadImplTest method testParentDataTypeToDataType.
@Test
public void testParentDataTypeToDataType() throws Exception {
assertEquals(Lists.newArrayList(), familiesRead.parentTypeToCurrentType().apply(Optional.empty()));
assertEquals(Lists.newArrayList(), familiesRead.parentTypeToCurrentType().apply(Optional.of(Row.getDefaultInstance())));
final Family family = Family.getDefaultInstance();
final Row row = Row.newBuilder().addFamilies(family).build();
assertEquals(ImmutableList.of(family), familiesRead.parentTypeToCurrentType().apply(Optional.of(row)));
}
Aggregations