Examples with NameBytesPair org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair used on opensource projects

Example 6 with NameBytesPair

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair in project hbase by apache.

the class TestVisibilityLabelsWithDefaultVisLabelService method testAddVisibilityLabelsOnRSRestart.

@Test
public void testAddVisibilityLabelsOnRSRestart() throws Exception {
    List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster().getRegionServerThreads();
    for (RegionServerThread rsThread : regionServerThreads) {
        rsThread.getRegionServer().abort("Aborting ");
    }
    // Start one new RS
    RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer();
    waitForLabelsRegionAvailability(rs.getRegionServer());
    final AtomicBoolean vcInitialized = new AtomicBoolean(true);
    do {
        PrivilegedExceptionAction<VisibilityLabelsResponse> action = new PrivilegedExceptionAction<VisibilityLabelsResponse>() {

            @Override
            public VisibilityLabelsResponse run() throws Exception {
                String[] labels = { SECRET, CONFIDENTIAL, PRIVATE, "ABC", "XYZ" };
                try (Connection conn = ConnectionFactory.createConnection(conf)) {
                    VisibilityLabelsResponse resp = VisibilityClient.addLabels(conn, labels);
                    List<RegionActionResult> results = resp.getResultList();
                    if (results.get(0).hasException()) {
                        NameBytesPair pair = results.get(0).getException();
                        Throwable t = ProtobufUtil.toException(pair);
                        LOG.debug("Got exception writing labels", t);
                        if (t instanceof VisibilityControllerNotReadyException) {
                            vcInitialized.set(false);
                            LOG.warn("VisibilityController was not yet initialized");
                            Threads.sleep(10);
                        } else {
                            vcInitialized.set(true);
                        }
                    } else
                        LOG.debug("new labels added: " + resp);
                } catch (Throwable t) {
                    throw new IOException(t);
                }
                return null;
            }
        };
        SUPERUSER.runAs(action);
    } while (!vcInitialized.get());
    // Scan the visibility label
    Scan s = new Scan();
    s.setAuthorizations(new Authorizations(VisibilityUtils.SYSTEM_LABEL));
    int i = 0;
    try (Table ht = TEST_UTIL.getConnection().getTable(LABELS_TABLE_NAME);
        ResultScanner scanner = ht.getScanner(s)) {
        while (true) {
            Result next = scanner.next();
            if (next == null) {
                break;
            }
            i++;
        }
    }
    // One label is the "system" label.
    Assert.assertEquals("The count should be 13", 13, i);
}

Example 7 with NameBytesPair

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair in project hbase by apache.

the class RSRpcServices method doNonAtomicRegionMutation.

/**
   * Run through the regionMutation <code>rm</code> and per Mutation, do the work, and then when
   * done, add an instance of a {@link ResultOrException} that corresponds to each Mutation.
   * @param region
   * @param actions
   * @param cellScanner
   * @param builder
   * @param cellsToReturn  Could be null. May be allocated in this method.  This is what this
   * method returns as a 'result'.
   * @param closeCallBack the callback to be used with multigets
   * @param context the current RpcCallContext
   * @return Return the <code>cellScanner</code> passed
   */
private List<CellScannable> doNonAtomicRegionMutation(final Region region, final OperationQuota quota, final RegionAction actions, final CellScanner cellScanner, final RegionActionResult.Builder builder, List<CellScannable> cellsToReturn, long nonceGroup, final RegionScannersCloseCallBack closeCallBack, RpcCallContext context) {
    // Gather up CONTIGUOUS Puts and Deletes in this mutations List.  Idea is that rather than do
    // one at a time, we instead pass them in batch.  Be aware that the corresponding
    // ResultOrException instance that matches each Put or Delete is then added down in the
    // doBatchOp call.  We should be staying aligned though the Put and Delete are deferred/batched
    List<ClientProtos.Action> mutations = null;
    long maxQuotaResultSize = Math.min(maxScannerResultSize, quota.getReadAvailable());
    IOException sizeIOE = null;
    Object lastBlock = null;
    ClientProtos.ResultOrException.Builder resultOrExceptionBuilder = ResultOrException.newBuilder();
    boolean hasResultOrException = false;
    for (ClientProtos.Action action : actions.getActionList()) {
        hasResultOrException = false;
        resultOrExceptionBuilder.clear();
        try {
            Result r = null;
            if (context != null && context.isRetryImmediatelySupported() && (context.getResponseCellSize() > maxQuotaResultSize || context.getResponseBlockSize() + context.getResponseExceptionSize() > maxQuotaResultSize)) {
                // change after the response size limit is reached.
                if (sizeIOE == null) {
                    // We don't need the stack un-winding do don't throw the exception.
                    // Throwing will kill the JVM's JIT.
                    //
                    // Instead just create the exception and then store it.
                    sizeIOE = new MultiActionResultTooLarge("Max size exceeded" + " CellSize: " + context.getResponseCellSize() + " BlockSize: " + context.getResponseBlockSize());
                    // Only report the exception once since there's only one request that
                    // caused the exception. Otherwise this number will dominate the exceptions count.
                    rpcServer.getMetrics().exception(sizeIOE);
                }
                // Now that there's an exception is known to be created
                // use it for the response.
                //
                // This will create a copy in the builder.
                hasResultOrException = true;
                NameBytesPair pair = ResponseConverter.buildException(sizeIOE);
                resultOrExceptionBuilder.setException(pair);
                context.incrementResponseExceptionSize(pair.getSerializedSize());
                resultOrExceptionBuilder.setIndex(action.getIndex());
                builder.addResultOrException(resultOrExceptionBuilder.build());
                if (cellScanner != null) {
                    skipCellsForMutation(action, cellScanner);
                }
                continue;
            }
            if (action.hasGet()) {
                long before = EnvironmentEdgeManager.currentTime();
                try {
                    Get get = ProtobufUtil.toGet(action.getGet());
                    if (context != null) {
                        r = get(get, ((HRegion) region), closeCallBack, context);
                    } else {
                        r = region.get(get);
                    }
                } finally {
                    if (regionServer.metricsRegionServer != null) {
                        regionServer.metricsRegionServer.updateGet(EnvironmentEdgeManager.currentTime() - before);
                    }
                }
            } else if (action.hasServiceCall()) {
                hasResultOrException = true;
                try {
                    com.google.protobuf.Message result = execServiceOnRegion(region, action.getServiceCall());
                    ClientProtos.CoprocessorServiceResult.Builder serviceResultBuilder = ClientProtos.CoprocessorServiceResult.newBuilder();
                    resultOrExceptionBuilder.setServiceResult(serviceResultBuilder.setValue(serviceResultBuilder.getValueBuilder().setName(result.getClass().getName()).setValue(UnsafeByteOperations.unsafeWrap(result.toByteArray()))));
                } catch (IOException ioe) {
                    rpcServer.getMetrics().exception(ioe);
                    NameBytesPair pair = ResponseConverter.buildException(ioe);
                    resultOrExceptionBuilder.setException(pair);
                    context.incrementResponseExceptionSize(pair.getSerializedSize());
                }
            } else if (action.hasMutation()) {
                MutationType type = action.getMutation().getMutateType();
                if (type != MutationType.PUT && type != MutationType.DELETE && mutations != null && !mutations.isEmpty()) {
                    // Flush out any Puts or Deletes already collected.
                    doBatchOp(builder, region, quota, mutations, cellScanner);
                    mutations.clear();
                }
                switch(type) {
                    case APPEND:
                        r = append(region, quota, action.getMutation(), cellScanner, nonceGroup);
                        break;
                    case INCREMENT:
                        r = increment(region, quota, action.getMutation(), cellScanner, nonceGroup);
                        break;
                    case PUT:
                    case DELETE:
                        // Collect the individual mutations and apply in a batch
                        if (mutations == null) {
                            mutations = new ArrayList<>(actions.getActionCount());
                        }
                        mutations.add(action);
                        break;
                    default:
                        throw new DoNotRetryIOException("Unsupported mutate type: " + type.name());
                }
            } else {
                throw new HBaseIOException("Unexpected Action type");
            }
            if (r != null) {
                ClientProtos.Result pbResult = null;
                if (isClientCellBlockSupport(context)) {
                    pbResult = ProtobufUtil.toResultNoData(r);
                    //  Hard to guess the size here.  Just make a rough guess.
                    if (cellsToReturn == null) {
                        cellsToReturn = new ArrayList<>();
                    }
                    cellsToReturn.add(r);
                } else {
                    pbResult = ProtobufUtil.toResult(r);
                }
                lastBlock = addSize(context, r, lastBlock);
                hasResultOrException = true;
                resultOrExceptionBuilder.setResult(pbResult);
            }
        // Could get to here and there was no result and no exception.  Presumes we added
        // a Put or Delete to the collecting Mutations List for adding later.  In this
        // case the corresponding ResultOrException instance for the Put or Delete will be added
        // down in the doBatchOp method call rather than up here.
        } catch (IOException ie) {
            rpcServer.getMetrics().exception(ie);
            hasResultOrException = true;
            NameBytesPair pair = ResponseConverter.buildException(ie);
            resultOrExceptionBuilder.setException(pair);
            context.incrementResponseExceptionSize(pair.getSerializedSize());
        }
        if (hasResultOrException) {
            // Propagate index.
            resultOrExceptionBuilder.setIndex(action.getIndex());
            builder.addResultOrException(resultOrExceptionBuilder.build());
        }
    }
    // Finish up any outstanding mutations
    if (mutations != null && !mutations.isEmpty()) {
        doBatchOp(builder, region, quota, mutations, cellScanner);
    }
    return cellsToReturn;
}

Example 8 with NameBytesPair

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair in project hbase by apache.

the class ProtobufUtil method toGet.

/**
 * Convert a protocol buffer Get to a client Get
 *
 * @param proto the protocol buffer Get to convert
 * @return the converted client Get
 * @throws IOException
 */
public static Get toGet(final ClientProtos.Get proto) throws IOException {
    if (proto == null)
        return null;
    byte[] row = proto.getRow().toByteArray();
    Get get = new Get(row);
    if (proto.hasCacheBlocks()) {
        get.setCacheBlocks(proto.getCacheBlocks());
    }
    if (proto.hasMaxVersions()) {
        get.readVersions(proto.getMaxVersions());
    }
    if (proto.hasStoreLimit()) {
        get.setMaxResultsPerColumnFamily(proto.getStoreLimit());
    }
    if (proto.hasStoreOffset()) {
        get.setRowOffsetPerColumnFamily(proto.getStoreOffset());
    }
    if (proto.getCfTimeRangeCount() > 0) {
        for (HBaseProtos.ColumnFamilyTimeRange cftr : proto.getCfTimeRangeList()) {
            TimeRange timeRange = toTimeRange(cftr.getTimeRange());
            get.setColumnFamilyTimeRange(cftr.getColumnFamily().toByteArray(), timeRange.getMin(), timeRange.getMax());
        }
    }
    if (proto.hasTimeRange()) {
        TimeRange timeRange = toTimeRange(proto.getTimeRange());
        get.setTimeRange(timeRange.getMin(), timeRange.getMax());
    }
    if (proto.hasFilter()) {
        FilterProtos.Filter filter = proto.getFilter();
        get.setFilter(ProtobufUtil.toFilter(filter));
    }
    for (NameBytesPair attribute : proto.getAttributeList()) {
        get.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
    }
    if (proto.getColumnCount() > 0) {
        for (Column column : proto.getColumnList()) {
            byte[] family = column.getFamily().toByteArray();
            if (column.getQualifierCount() > 0) {
                for (ByteString qualifier : column.getQualifierList()) {
                    get.addColumn(family, qualifier.toByteArray());
                }
            } else {
                get.addFamily(family);
            }
        }
    }
    if (proto.hasExistenceOnly() && proto.getExistenceOnly()) {
        get.setCheckExistenceOnly(true);
    }
    if (proto.hasConsistency()) {
        get.setConsistency(toConsistency(proto.getConsistency()));
    }
    if (proto.hasLoadColumnFamiliesOnDemand()) {
        get.setLoadColumnFamiliesOnDemand(proto.getLoadColumnFamiliesOnDemand());
    }
    return get;
}

Example 9 with NameBytesPair

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair in project hbase by apache.

the class ProtobufUtil method toPut.

/**
 * Convert a protocol buffer Mutate to a Put.
 *
 * @param proto The protocol buffer MutationProto to convert
 * @param cellScanner If non-null, the Cell data that goes with this proto.
 * @return A client Put.
 * @throws IOException
 */
public static Put toPut(final MutationProto proto, final CellScanner cellScanner) throws IOException {
    // TODO: Server-side at least why do we convert back to the Client types?  Why not just pb it?
    MutationType type = proto.getMutateType();
    assert type == MutationType.PUT : type.name();
    long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
    Put put = proto.hasRow() ? new Put(proto.getRow().toByteArray(), timestamp) : null;
    int cellCount = proto.hasAssociatedCellCount() ? proto.getAssociatedCellCount() : 0;
    if (cellCount > 0) {
        // The proto has metadata only and the data is separate to be found in the cellScanner.
        if (cellScanner == null) {
            throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " + toShortString(proto));
        }
        for (int i = 0; i < cellCount; i++) {
            if (!cellScanner.advance()) {
                throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i + " no cell returned: " + toShortString(proto));
            }
            Cell cell = cellScanner.current();
            if (put == null) {
                put = new Put(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
            }
            put.add(cell);
        }
    } else {
        if (put == null) {
            throw new IllegalArgumentException("row cannot be null");
        }
        // The proto has the metadata and the data itself
        ExtendedCellBuilder cellBuilder = ExtendedCellBuilderFactory.create(CellBuilderType.SHALLOW_COPY);
        for (ColumnValue column : proto.getColumnValueList()) {
            byte[] family = column.getFamily().toByteArray();
            for (QualifierValue qv : column.getQualifierValueList()) {
                if (!qv.hasValue()) {
                    throw new DoNotRetryIOException("Missing required field: qualifier value");
                }
                long ts = timestamp;
                if (qv.hasTimestamp()) {
                    ts = qv.getTimestamp();
                }
                byte[] allTagsBytes;
                if (qv.hasTags()) {
                    allTagsBytes = qv.getTags().toByteArray();
                    if (qv.hasDeleteType()) {
                        put.add(cellBuilder.clear().setRow(proto.getRow().toByteArray()).setFamily(family).setQualifier(qv.hasQualifier() ? qv.getQualifier().toByteArray() : null).setTimestamp(ts).setType(fromDeleteType(qv.getDeleteType()).getCode()).setTags(allTagsBytes).build());
                    } else {
                        put.add(cellBuilder.clear().setRow(put.getRow()).setFamily(family).setQualifier(qv.hasQualifier() ? qv.getQualifier().toByteArray() : null).setTimestamp(ts).setType(Cell.Type.Put).setValue(qv.hasValue() ? qv.getValue().toByteArray() : null).setTags(allTagsBytes).build());
                    }
                } else {
                    if (qv.hasDeleteType()) {
                        put.add(cellBuilder.clear().setRow(put.getRow()).setFamily(family).setQualifier(qv.hasQualifier() ? qv.getQualifier().toByteArray() : null).setTimestamp(ts).setType(fromDeleteType(qv.getDeleteType()).getCode()).build());
                    } else {
                        put.add(cellBuilder.clear().setRow(put.getRow()).setFamily(family).setQualifier(qv.hasQualifier() ? qv.getQualifier().toByteArray() : null).setTimestamp(ts).setType(Type.Put).setValue(qv.hasValue() ? qv.getValue().toByteArray() : null).build());
                    }
                }
            }
        }
    }
    put.setDurability(toDurability(proto.getDurability()));
    for (NameBytesPair attribute : proto.getAttributeList()) {
        put.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
    }
    return put;
}

Example 10 with NameBytesPair

use of org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos.NameBytesPair in project hbase by apache.

the class ProtobufUtil method toDelete.

/**
 * Convert a protocol buffer Mutate to a Delete
 *
 * @param proto the protocol buffer Mutate to convert
 * @param cellScanner if non-null, the data that goes with this delete.
 * @return the converted client Delete
 * @throws IOException
 */
public static Delete toDelete(final MutationProto proto, final CellScanner cellScanner) throws IOException {
    MutationType type = proto.getMutateType();
    assert type == MutationType.DELETE : type.name();
    long timestamp = proto.hasTimestamp() ? proto.getTimestamp() : HConstants.LATEST_TIMESTAMP;
    Delete delete = proto.hasRow() ? new Delete(proto.getRow().toByteArray(), timestamp) : null;
    int cellCount = proto.hasAssociatedCellCount() ? proto.getAssociatedCellCount() : 0;
    if (cellCount > 0) {
        // The proto has metadata only and the data is separate to be found in the cellScanner.
        if (cellScanner == null) {
            // TextFormat should be fine for a Delete since it carries no data, just coordinates.
            throw new DoNotRetryIOException("Cell count of " + cellCount + " but no cellScanner: " + TextFormat.shortDebugString(proto));
        }
        for (int i = 0; i < cellCount; i++) {
            if (!cellScanner.advance()) {
                // TextFormat should be fine for a Delete since it carries no data, just coordinates.
                throw new DoNotRetryIOException("Cell count of " + cellCount + " but at index " + i + " no cell returned: " + TextFormat.shortDebugString(proto));
            }
            Cell cell = cellScanner.current();
            if (delete == null) {
                delete = new Delete(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength(), timestamp);
            }
            delete.add(cell);
        }
    } else {
        if (delete == null) {
            throw new IllegalArgumentException("row cannot be null");
        }
        for (ColumnValue column : proto.getColumnValueList()) {
            byte[] family = column.getFamily().toByteArray();
            for (QualifierValue qv : column.getQualifierValueList()) {
                DeleteType deleteType = qv.getDeleteType();
                byte[] qualifier = null;
                if (qv.hasQualifier()) {
                    qualifier = qv.getQualifier().toByteArray();
                }
                long ts = cellTimestampOrLatest(qv);
                if (deleteType == DeleteType.DELETE_ONE_VERSION) {
                    delete.addColumn(family, qualifier, ts);
                } else if (deleteType == DeleteType.DELETE_MULTIPLE_VERSIONS) {
                    delete.addColumns(family, qualifier, ts);
                } else if (deleteType == DeleteType.DELETE_FAMILY_VERSION) {
                    delete.addFamilyVersion(family, ts);
                } else {
                    delete.addFamily(family, ts);
                }
            }
        }
    }
    delete.setDurability(toDurability(proto.getDurability()));
    for (NameBytesPair attribute : proto.getAttributeList()) {
        delete.setAttribute(attribute.getName(), attribute.getValue().toByteArray());
    }
    return delete;
}