Search in sources :

Example 6 with Row

use of com.google.api.ads.admanager.jaxws.v202202.Row in project beam by apache.

the class BigtableServiceImplTest method testRead.

/**
 * This test ensures that protobuf creation and interactions with {@link BigtableDataClient} work
 * as expected.
 *
 * @throws IOException
 * @throws InterruptedException
 */
@Test
public void testRead() throws IOException {
    ByteKey start = ByteKey.copyFrom("a".getBytes(StandardCharsets.UTF_8));
    ByteKey end = ByteKey.copyFrom("b".getBytes(StandardCharsets.UTF_8));
    when(mockBigtableSource.getRanges()).thenReturn(Arrays.asList(ByteKeyRange.of(start, end)));
    when(mockBigtableSource.getTableId()).thenReturn(StaticValueProvider.of(TABLE_ID));
    @SuppressWarnings("unchecked") ResultScanner<Row> mockResultScanner = Mockito.mock(ResultScanner.class);
    Row expectedRow = Row.newBuilder().setKey(ByteString.copyFromUtf8("a")).build();
    when(mockResultScanner.next()).thenReturn(expectedRow).thenReturn(null);
    when(mockBigtableDataClient.readRows(any(ReadRowsRequest.class))).thenReturn(mockResultScanner);
    BigtableService.Reader underTest = new BigtableServiceImpl.BigtableReaderImpl(mockSession, mockBigtableSource);
    underTest.start();
    Assert.assertEquals(expectedRow, underTest.getCurrentRow());
    Assert.assertFalse(underTest.advance());
    underTest.close();
    verify(mockResultScanner, times(1)).close();
    verifyMetricWasSet("google.bigtable.v2.ReadRows", "ok", 1);
}
Also used : ByteKey(org.apache.beam.sdk.io.range.ByteKey) ReadRowsRequest(com.google.bigtable.v2.ReadRowsRequest) Row(com.google.bigtable.v2.Row) Test(org.junit.Test)

Example 7 with Row

use of com.google.api.ads.admanager.jaxws.v202202.Row in project YCSB by brianfrankcooper.

the class GoogleBigtableClient method scan.

@Override
public Status scan(String table, String startkey, int recordcount, Set<String> fields, Vector<HashMap<String, ByteIterator>> result) {
    setTable(table);
    RowFilter filter = RowFilter.newBuilder().setFamilyNameRegexFilterBytes(ByteStringer.wrap(columnFamilyBytes)).build();
    if (fields != null && fields.size() > 0) {
        Builder filterChain = RowFilter.Chain.newBuilder();
        filterChain.addFilters(filter);
        filterChain.addFilters(RowFilter.newBuilder().setCellsPerColumnLimitFilter(1).build());
        int count = 0;
        // usually "field#" so pre-alloc
        final StringBuilder regex = new StringBuilder(fields.size() * 6);
        for (final String field : fields) {
            if (count++ > 0) {
                regex.append("|");
            }
            regex.append(field);
        }
        filterChain.addFilters(RowFilter.newBuilder().setColumnQualifierRegexFilter(ByteStringer.wrap(regex.toString().getBytes()))).build();
        filter = RowFilter.newBuilder().setChain(filterChain.build()).build();
    }
    final RowRange range = RowRange.newBuilder().setStartKeyClosed(ByteStringer.wrap(startkey.getBytes())).build();
    final RowSet rowSet = RowSet.newBuilder().addRowRanges(range).build();
    final ReadRowsRequest.Builder rrr = ReadRowsRequest.newBuilder().setTableNameBytes(ByteStringer.wrap(lastTableBytes)).setFilter(filter).setRows(rowSet);
    List<Row> rows;
    try {
        rows = client.readRowsAsync(rrr.build()).get();
        if (rows == null || rows.isEmpty()) {
            return Status.NOT_FOUND;
        }
        int numResults = 0;
        for (final Row row : rows) {
            final HashMap<String, ByteIterator> rowResult = new HashMap<String, ByteIterator>(fields != null ? fields.size() : 10);
            for (final Family family : row.getFamiliesList()) {
                if (Arrays.equals(family.getNameBytes().toByteArray(), columnFamilyBytes)) {
                    for (final Column column : family.getColumnsList()) {
                        // we should only have a single cell per column
                        rowResult.put(column.getQualifier().toString(UTF8_CHARSET), new ByteArrayByteIterator(column.getCells(0).getValue().toByteArray()));
                        if (debug) {
                            System.out.println("Result for field: " + column.getQualifier().toString(UTF8_CHARSET) + " is: " + column.getCells(0).getValue().toString(UTF8_CHARSET));
                        }
                    }
                }
            }
            result.add(rowResult);
            numResults++;
            if (numResults >= recordcount) {
                // if hit recordcount, bail out
                break;
            }
        }
        return Status.OK;
    } catch (InterruptedException e) {
        System.err.println("Interrupted during scan: " + e);
        Thread.currentThread().interrupt();
        return Status.ERROR;
    } catch (ExecutionException e) {
        System.err.println("Exception during scan: " + e);
        return Status.ERROR;
    }
}
Also used : HashMap(java.util.HashMap) Builder(com.google.bigtable.v2.RowFilter.Chain.Builder) RowSet(com.google.bigtable.v2.RowSet) ReadRowsRequest(com.google.bigtable.v2.ReadRowsRequest) ByteString(com.google.protobuf.ByteString) ByteArrayByteIterator(site.ycsb.ByteArrayByteIterator) RowFilter(com.google.bigtable.v2.RowFilter) ByteIterator(site.ycsb.ByteIterator) ByteArrayByteIterator(site.ycsb.ByteArrayByteIterator) RowRange(com.google.bigtable.v2.RowRange) Column(com.google.bigtable.v2.Column) Family(com.google.bigtable.v2.Family) Row(com.google.bigtable.v2.Row) DeleteFromRow(com.google.bigtable.v2.Mutation.DeleteFromRow) ExecutionException(java.util.concurrent.ExecutionException)

Example 8 with Row

use of com.google.api.ads.admanager.jaxws.v202202.Row in project BibleMultiConverter by schierlm.

the class USX3 method doImportBook.

@Override
protected ParatextBook doImportBook(File inputFile) throws Exception {
    if (!inputFile.getName().toLowerCase().endsWith(".usx"))
        return null;
    ValidateXML.validateFileBeforeParsing(getSchema(), inputFile);
    JAXBContext ctx = JAXBContext.newInstance(ObjectFactory.class.getPackage().getName());
    XMLInputFactory xif = XMLInputFactory.newFactory();
    XMLStreamReader xsr = xif.createXMLStreamReader(new FileInputStream(inputFile));
    Unmarshaller u = ctx.createUnmarshaller();
    u.setListener(unmarshallerLocationListener);
    unmarshallerLocationListener.setXMLStreamReader(inputFile.getName(), xsr);
    Usx doc = (Usx) u.unmarshal(xsr);
    xsr.close();
    ParatextBook.ParatextID id = ParatextBook.ParatextID.fromIdentifier(doc.getBook().getCode().toUpperCase());
    if (id == null) {
        System.out.println("WARNING: Skipping book with unknown ID: " + doc.getBook().getCode());
        return null;
    }
    ParatextBook result = new ParatextBook(id, doc.getBook().getContent());
    ParatextCharacterContent charContent = null;
    for (Object o : doc.getParaOrTableOrChapter()) {
        if (o instanceof Para) {
            Para para = (Para) o;
            if (BOOK_HEADER_ATTRIBUTE_TAGS.contains(para.getStyle().value())) {
                String value = "";
                for (Object oo : para.getContent()) {
                    if (oo instanceof String) {
                        value += ((String) oo).replaceAll("[ \r\n\t]+", " ");
                    } else {
                        throw new RuntimeException("Unsupported content in attribute: " + oo.getClass());
                    }
                }
                result.getAttributes().put(para.getStyle().value(), value);
                charContent = null;
            } else if (para.getStyle() == ParaStyle.PB) {
                if (charContent == null) {
                    charContent = new ParatextCharacterContent();
                    result.getContent().add(charContent);
                }
                charContent.getContent().add(new ParatextCharacterContent.AutoClosingFormatting(ParatextCharacterContent.AutoClosingFormattingKind.PAGE_BREAK, false));
            } else if (PARA_STYLE_UNSUPPORTED.contains(para.getStyle())) {
                // skip
                charContent = null;
            } else {
                result.getContent().add(new ParatextBook.ParagraphStart(PARA_STYLE_MAP.get(para.getStyle())));
                charContent = null;
                if (!para.getContent().isEmpty()) {
                    charContent = new ParatextCharacterContent();
                    result.getContent().add(charContent);
                    parseCharContent(para.getContent(), charContent);
                }
            }
        } else if (o instanceof Table) {
            Table table = (Table) o;
            for (Row row : table.getRow()) {
                result.getContent().add(new ParatextBook.ParagraphStart(ParatextBook.ParagraphKind.TABLE_ROW));
                for (Object oo : row.getVerseOrCell()) {
                    if (oo instanceof Verse) {
                        Verse verse = (Verse) oo;
                        ParatextCharacterContent.ParatextCharacterContentPart verseStartOrEnd = handleVerse(verse);
                        charContent = new ParatextCharacterContent();
                        result.getContent().add(charContent);
                        charContent.getContent().add(verseStartOrEnd);
                    } else if (oo instanceof Cell) {
                        Cell cell = (Cell) oo;
                        result.getContent().add(new ParatextBook.TableCellStart(cell.getStyle().value()));
                        charContent = new ParatextCharacterContent();
                        result.getContent().add(charContent);
                        parseCharContent(cell.getContent(), charContent);
                    } else {
                        throw new IOException("Unsupported table row element: " + o.getClass().getName());
                    }
                }
            }
            charContent = null;
        } else if (o instanceof Chapter) {
            Chapter chapter = (Chapter) o;
            if (chapter.getSid() != null) {
                // Assume start chapter
                result.getContent().add(new ParatextBook.ChapterStart(new ChapterIdentifier(result.getId(), ((Chapter) o).getNumber().intValue())));
            } else if (chapter.getEid() != null) {
                // Assume end chapter
                ChapterIdentifier location = ChapterIdentifier.fromLocationString(chapter.getEid());
                if (location == null) {
                    throw new IOException("Invalid chapter eid found: " + chapter.getEid());
                }
                result.getContent().add(new ParatextBook.ChapterEnd(location));
            } else {
                throw new IOException("Invalid chapter found, both sid and eid are undefined: " + chapter);
            }
            charContent = null;
        } else if (o instanceof Note) {
            if (charContent == null) {
                charContent = new ParatextCharacterContent();
                result.getContent().add(charContent);
            }
            Note note = (Note) o;
            ParatextCharacterContent.FootnoteXref nx = new ParatextCharacterContent.FootnoteXref(NOTE_STYLE_MAP.get(note.getStyle()), note.getCaller());
            charContent.getContent().add(nx);
            parseCharContent(note.getContent(), nx);
        } else if (o instanceof Sidebar) {
            System.out.println("WARNING: Skipping sidebar (study bible content)");
            charContent = null;
        } else {
            throw new IOException("Unsupported book level element: " + o.getClass().getName());
        }
    }
    return result;
}
Also used : XMLStreamReader(javax.xml.stream.XMLStreamReader) JAXBContext(javax.xml.bind.JAXBContext) Unmarshaller(javax.xml.bind.Unmarshaller) Cell(biblemulticonverter.schema.usx3.Cell) Table(biblemulticonverter.schema.usx3.Table) Para(biblemulticonverter.schema.usx3.Para) Chapter(biblemulticonverter.schema.usx3.Chapter) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) Note(biblemulticonverter.schema.usx3.Note) Usx(biblemulticonverter.schema.usx3.Usx) Row(biblemulticonverter.schema.usx3.Row) ChapterIdentifier(biblemulticonverter.format.paratext.model.ChapterIdentifier) XMLInputFactory(javax.xml.stream.XMLInputFactory) Verse(biblemulticonverter.schema.usx3.Verse) Sidebar(biblemulticonverter.schema.usx3.Sidebar)

Example 9 with Row

use of com.google.api.ads.admanager.jaxws.v202202.Row in project BibleMultiConverter by schierlm.

the class USX3 method doExportBook.

@Override
protected void doExportBook(ParatextBook book, File outFile) throws Exception {
    ObjectFactory of = new ObjectFactory();
    Usx usx = of.createUsx();
    usx.setVersion("3.0");
    usx.setBook(of.createBook());
    usx.getBook().setStyle("id");
    usx.getBook().setCode(book.getId().getIdentifier());
    usx.getBook().setContent(book.getBibleName());
    for (Map.Entry<String, String> attr : book.getAttributes().entrySet()) {
        Para para = new Para();
        para.setStyle(ParaStyle.fromValue(attr.getKey()));
        para.getContent().add(attr.getValue());
        usx.getParaOrTableOrChapter().add(para);
    }
    book.accept(new ParatextBook.ParatextBookContentVisitor<IOException>() {

        List<Object> currentContent = null;

        Table currentTable = null;

        @Override
        public void visitChapterStart(ChapterIdentifier location) throws IOException {
            Chapter ch = new Chapter();
            ch.setStyle("c");
            ch.setSid(location.toString());
            ch.setNumber(BigInteger.valueOf(location.chapter));
            usx.getParaOrTableOrChapter().add(ch);
            currentContent = null;
            currentTable = null;
        }

        @Override
        public void visitChapterEnd(ChapterIdentifier location) throws IOException {
            Chapter ch = new Chapter();
            ch.setEid(location.toString());
            usx.getParaOrTableOrChapter().add(ch);
            currentContent = null;
            currentTable = null;
        }

        @Override
        public void visitParagraphStart(ParatextBook.ParagraphKind kind) throws IOException {
            if (kind == ParatextBook.ParagraphKind.TABLE_ROW) {
                if (currentTable == null) {
                    currentTable = new Table();
                    usx.getParaOrTableOrChapter().add(currentTable);
                }
                Row row = new Row();
                row.setStyle("tr");
                currentTable.getRow().add(row);
                currentContent = currentTable.getRow().get(currentTable.getRow().size() - 1).getVerseOrCell();
            } else {
                Para para = new Para();
                para.setStyle(PARA_KIND_MAP.get(kind));
                usx.getParaOrTableOrChapter().add(para);
                currentContent = para.getContent();
                currentTable = null;
            }
        }

        @Override
        public void visitTableCellStart(String tag) throws IOException {
            if (currentTable == null) {
                System.out.println("WARNING: Table cell outside of table");
                return;
            }
            Row currentRow = currentTable.getRow().get(currentTable.getRow().size() - 1);
            Cell cell = new Cell();
            cell.setAlign(tag.contains("r") ? CellAlign.END : CellAlign.START);
            cell.setStyle(CellStyle.fromValue(tag));
            currentRow.getVerseOrCell().add(cell);
            currentContent = cell.getContent();
        }

        @Override
        public void visitParatextCharacterContent(ParatextCharacterContent content) throws IOException {
            if (currentContent == null)
                visitParagraphStart(ParatextBook.ParagraphKind.PARAGRAPH_P);
            content.accept(new USX3.USXCharacterContentVisitor(currentContent));
        }
    });
    JAXBContext ctx = JAXBContext.newInstance(ObjectFactory.class.getPackage().getName());
    Marshaller m = ctx.createMarshaller();
    if (!Boolean.getBoolean("biblemulticonverter.skipxmlvalidation"))
        m.setSchema(getSchema());
    m.marshal(usx, new UnifiedScriptureXMLWriter(new FileWriter(outFile), "UTF-8"));
}
Also used : Marshaller(javax.xml.bind.Marshaller) Table(biblemulticonverter.schema.usx3.Table) Para(biblemulticonverter.schema.usx3.Para) UnifiedScriptureXMLWriter(biblemulticonverter.format.paratext.utilities.UnifiedScriptureXMLWriter) FileWriter(java.io.FileWriter) Chapter(biblemulticonverter.schema.usx3.Chapter) JAXBContext(javax.xml.bind.JAXBContext) IOException(java.io.IOException) ObjectFactory(biblemulticonverter.schema.usx3.ObjectFactory) Usx(biblemulticonverter.schema.usx3.Usx) Row(biblemulticonverter.schema.usx3.Row) ChapterIdentifier(biblemulticonverter.format.paratext.model.ChapterIdentifier) Map(java.util.Map) EnumMap(java.util.EnumMap) Cell(biblemulticonverter.schema.usx3.Cell)

Example 10 with Row

use of com.google.api.ads.admanager.jaxws.v202202.Row in project simple-bigtable by spotify.

the class FamiliesReadImplTest method testParentDataTypeToDataType.

@Test
public void testParentDataTypeToDataType() throws Exception {
    assertEquals(Lists.newArrayList(), familiesRead.parentTypeToCurrentType().apply(Optional.empty()));
    assertEquals(Lists.newArrayList(), familiesRead.parentTypeToCurrentType().apply(Optional.of(Row.getDefaultInstance())));
    final Family family = Family.getDefaultInstance();
    final Row row = Row.newBuilder().addFamilies(family).build();
    assertEquals(ImmutableList.of(family), familiesRead.parentTypeToCurrentType().apply(Optional.of(row)));
}
Also used : Family(com.google.bigtable.v2.Family) Row(com.google.bigtable.v2.Row) Test(org.junit.Test)

Aggregations

Test (org.junit.Test)36 Row (com.google.bigtable.v2.Row)16 ByteString (com.google.protobuf.ByteString)11 Function (com.google.common.base.Function)8 Row (com.google.api.ads.admanager.axis.v202108.Row)6 Row (com.google.api.ads.admanager.axis.v202111.Row)6 Row (com.google.api.ads.admanager.axis.v202202.Row)6 Family (com.google.bigtable.v2.Family)6 ByteKey (org.apache.beam.sdk.io.range.ByteKey)6 Row (com.google.api.ads.admanager.axis.v202105.Row)5 Row (com.google.api.ads.admanager.jaxws.v202105.Row)5 Row (com.google.api.ads.admanager.jaxws.v202108.Row)5 Row (com.google.api.ads.admanager.jaxws.v202111.Row)5 Row (com.google.api.ads.admanager.jaxws.v202202.Row)5 ResultSet (com.google.api.ads.admanager.axis.v202108.ResultSet)4 ResultSet (com.google.api.ads.admanager.axis.v202111.ResultSet)4 ResultSet (com.google.api.ads.admanager.axis.v202202.ResultSet)4 Column (com.google.bigtable.v2.Column)4 ReadRowsRequest (com.google.bigtable.v2.ReadRowsRequest)4 ResultSet (com.google.api.ads.admanager.jaxws.v202105.ResultSet)3