Examples with ByteString org.apache.hbase.thirdparty.com.google.protobuf.ByteString used on opensource projects

Example 11 with ByteString

use of org.apache.hbase.thirdparty.com.google.protobuf.ByteString in project hbase by apache.

the class TestWALEntrySinkFilter method testWALEntryFilter.

/**
 * Test filter. Filter will filter out any write time that is <= 5 (BOUNDARY). We count how many
 * items we filter out and we count how many cells make it through for distribution way down below
 * in the Table#batch implementation. Puts in place a custom DevNullConnection so we can insert
 * our counting Table.
 * @throws IOException
 */
@Test
public void testWALEntryFilter() throws IOException {
    Configuration conf = HBaseConfiguration.create();
    // Make it so our filter is instantiated on construction of ReplicationSink.
    conf.setClass(DummyConnectionRegistry.REGISTRY_IMPL_CONF_KEY, DevNullConnectionRegistry.class, DummyConnectionRegistry.class);
    conf.setClass(WALEntrySinkFilter.WAL_ENTRY_FILTER_KEY, IfTimeIsGreaterThanBOUNDARYWALEntrySinkFilterImpl.class, WALEntrySinkFilter.class);
    conf.setClass(ClusterConnectionFactory.HBASE_SERVER_CLUSTER_CONNECTION_IMPL, DevNullAsyncClusterConnection.class, AsyncClusterConnection.class);
    ReplicationSink sink = new ReplicationSink(conf);
    // Create some dumb walentries.
    List<AdminProtos.WALEntry> entries = new ArrayList<>();
    AdminProtos.WALEntry.Builder entryBuilder = AdminProtos.WALEntry.newBuilder();
    // Need a tablename.
    ByteString tableName = ByteString.copyFromUtf8(TableName.valueOf(this.name.getMethodName()).toString());
    // Add WALEdit Cells to Cells List. The way edits arrive at the sink is with protos
    // describing the edit with all Cells from all edits aggregated in a single CellScanner.
    final List<Cell> cells = new ArrayList<>();
    int count = BOUNDARY * 2;
    for (int i = 0; i < count; i++) {
        byte[] bytes = Bytes.toBytes(i);
        // Create a wal entry. Everything is set to the current index as bytes or int/long.
        entryBuilder.clear();
        entryBuilder.setKey(entryBuilder.getKeyBuilder().setLogSequenceNumber(i).setEncodedRegionName(ByteString.copyFrom(bytes)).setWriteTime(i).setTableName(tableName).build());
        // Lets have one Cell associated with each WALEdit.
        entryBuilder.setAssociatedCellCount(1);
        entries.add(entryBuilder.build());
        // We need to add a Cell per WALEdit to the cells array.
        CellBuilder cellBuilder = CellBuilderFactory.create(CellBuilderType.DEEP_COPY);
        // Make cells whose row, family, cell, value, and ts are == 'i'.
        Cell cell = cellBuilder.setRow(bytes).setFamily(bytes).setQualifier(bytes).setType(Cell.Type.Put).setTimestamp(i).setValue(bytes).build();
        cells.add(cell);
    }
    // Now wrap our cells array in a CellScanner that we can pass in to replicateEntries. It has
    // all Cells from all the WALEntries made above.
    CellScanner cellScanner = new CellScanner() {

        // Set to -1 because advance gets called before current.
        int index = -1;

        @Override
        public Cell current() {
            return cells.get(index);
        }

        @Override
        public boolean advance() throws IOException {
            index++;
            return index < cells.size();
        }
    };
    // Call our sink.
    sink.replicateEntries(entries, cellScanner, null, null, null);
    // Check what made it through and what was filtered.
    assertTrue(FILTERED.get() > 0);
    assertTrue(UNFILTERED.get() > 0);
    assertEquals(count, FILTERED.get() + UNFILTERED.get());
}

Example 12 with ByteString

use of org.apache.hbase.thirdparty.com.google.protobuf.ByteString in project hbase by apache.

the class WALKeyImpl method getBuilder.

public WALProtos.WALKey.Builder getBuilder(WALCellCodec.ByteStringCompressor compressor) throws IOException {
    WALProtos.WALKey.Builder builder = WALProtos.WALKey.newBuilder();
    builder.setEncodedRegionName(compressor.compress(this.encodedRegionName, CompressionContext.DictionaryIndex.REGION));
    builder.setTableName(compressor.compress(this.tablename.getName(), CompressionContext.DictionaryIndex.TABLE));
    builder.setLogSequenceNumber(getSequenceId());
    builder.setWriteTime(writeTime);
    if (this.origLogSeqNum > 0) {
        builder.setOrigSequenceNumber(this.origLogSeqNum);
    }
    if (this.nonce != HConstants.NO_NONCE) {
        builder.setNonce(nonce);
    }
    if (this.nonceGroup != HConstants.NO_NONCE) {
        builder.setNonceGroup(nonceGroup);
    }
    HBaseProtos.UUID.Builder uuidBuilder = HBaseProtos.UUID.newBuilder();
    for (UUID clusterId : clusterIds) {
        uuidBuilder.setLeastSigBits(clusterId.getLeastSignificantBits());
        uuidBuilder.setMostSigBits(clusterId.getMostSignificantBits());
        builder.addClusterIds(uuidBuilder.build());
    }
    if (replicationScope != null) {
        for (Map.Entry<byte[], Integer> e : replicationScope.entrySet()) {
            ByteString family = compressor.compress(e.getKey(), CompressionContext.DictionaryIndex.FAMILY);
            builder.addScopes(FamilyScope.newBuilder().setFamily(family).setScopeType(ScopeType.forNumber(e.getValue())));
        }
    }
    if (extendedAttributes != null) {
        for (Map.Entry<String, byte[]> e : extendedAttributes.entrySet()) {
            WALProtos.Attribute attr = WALProtos.Attribute.newBuilder().setKey(e.getKey()).setValue(compressor.compress(e.getValue(), CompressionContext.DictionaryIndex.TABLE)).build();
            builder.addExtendedAttributes(attr);
        }
    }
    return builder;
}

Example 13 with ByteString

use of org.apache.hbase.thirdparty.com.google.protobuf.ByteString in project hbase by apache.

the class TestVisibilityLabelsWithDefaultVisLabelService method testListLabelsWithRegEx.

@Test
public void testListLabelsWithRegEx() throws Throwable {
    PrivilegedExceptionAction<ListLabelsResponse> action = new PrivilegedExceptionAction<ListLabelsResponse>() {

        @Override
        public ListLabelsResponse run() throws Exception {
            ListLabelsResponse response = null;
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                response = VisibilityClient.listLabels(conn, ".*secret");
            } catch (Throwable e) {
                throw new IOException(e);
            }
            // Only return the labels that end with 'secret'
            List<ByteString> labels = response.getLabelList();
            assertEquals(2, labels.size());
            assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(SECRET))));
            assertTrue(labels.contains(ByteString.copyFrom(Bytes.toBytes(TOPSECRET))));
            return null;
        }
    };
    SUPERUSER.runAs(action);
}

Example 14 with ByteString

use of org.apache.hbase.thirdparty.com.google.protobuf.ByteString in project hbase by apache.

the class TestVisibilityLabels method testClearUserAuths.

@Test
public void testClearUserAuths() throws Throwable {
    PrivilegedExceptionAction<Void> action = new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            String[] auths = { SECRET, CONFIDENTIAL, PRIVATE };
            String user = "testUser";
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                VisibilityClient.setAuths(conn, auths, user);
            } catch (Throwable e) {
                throw new IOException(e);
            }
            // Removing the auths for SECRET and CONFIDENTIAL for the user.
            // Passing a non existing auth also.
            auths = new String[] { SECRET, PUBLIC, CONFIDENTIAL };
            VisibilityLabelsResponse response = null;
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                response = VisibilityClient.clearAuths(conn, auths, user);
            } catch (Throwable e) {
                fail("Should not have failed");
            }
            List<RegionActionResult> resultList = response.getResultList();
            assertEquals(3, resultList.size());
            assertTrue(resultList.get(0).getException().getValue().isEmpty());
            assertEquals("org.apache.hadoop.hbase.DoNotRetryIOException", resultList.get(1).getException().getName());
            assertTrue(Bytes.toString(resultList.get(1).getException().getValue().toByteArray()).contains("org.apache.hadoop.hbase.security.visibility.InvalidLabelException: " + "Label 'public' is not set for the user testUser"));
            assertTrue(resultList.get(2).getException().getValue().isEmpty());
            try (Connection connection = ConnectionFactory.createConnection(conf);
                Table ht = connection.getTable(LABELS_TABLE_NAME)) {
                ResultScanner scanner = ht.getScanner(new Scan());
                Result result = null;
                List<Result> results = new ArrayList<>();
                while ((result = scanner.next()) != null) {
                    results.add(result);
                }
                List<String> curAuths = extractAuths(user, results);
                assertTrue(curAuths.contains(PRIVATE));
                assertEquals(1, curAuths.size());
            }
            GetAuthsResponse authsResponse = null;
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                authsResponse = VisibilityClient.getAuths(conn, user);
            } catch (Throwable e) {
                throw new IOException(e);
            }
            List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
            for (ByteString authBS : authsResponse.getAuthList()) {
                authsList.add(Bytes.toString(authBS.toByteArray()));
            }
            assertEquals(1, authsList.size());
            assertTrue(authsList.contains(PRIVATE));
            return null;
        }
    };
    SUPERUSER.runAs(action);
}

Example 15 with ByteString

use of org.apache.hbase.thirdparty.com.google.protobuf.ByteString in project hbase by apache.

the class TestWithDisabledAuthorization method testManageUserAuths.

@Test
public void testManageUserAuths() throws Throwable {
    // Even though authorization is disabled, we should be able to manage user auths
    SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                VisibilityClient.setAuths(conn, new String[] { SECRET, CONFIDENTIAL }, USER_RW.getShortName());
            } catch (Throwable t) {
                fail("Should not have failed");
            }
            return null;
        }
    });
    PrivilegedExceptionAction<List<String>> getAuths = new PrivilegedExceptionAction<List<String>>() {

        @Override
        public List<String> run() throws Exception {
            GetAuthsResponse authsResponse = null;
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                authsResponse = VisibilityClient.getAuths(conn, USER_RW.getShortName());
            } catch (Throwable t) {
                fail("Should not have failed");
            }
            List<String> authsList = new ArrayList<>(authsResponse.getAuthList().size());
            for (ByteString authBS : authsResponse.getAuthList()) {
                authsList.add(Bytes.toString(authBS.toByteArray()));
            }
            return authsList;
        }
    };
    List<String> authsList = SUPERUSER.runAs(getAuths);
    assertEquals(2, authsList.size());
    assertTrue(authsList.contains(SECRET));
    assertTrue(authsList.contains(CONFIDENTIAL));
    SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                VisibilityClient.clearAuths(conn, new String[] { SECRET }, USER_RW.getShortName());
            } catch (Throwable t) {
                fail("Should not have failed");
            }
            return null;
        }
    });
    authsList = SUPERUSER.runAs(getAuths);
    assertEquals(1, authsList.size());
    assertTrue(authsList.contains(CONFIDENTIAL));
    SUPERUSER.runAs(new PrivilegedExceptionAction<Void>() {

        @Override
        public Void run() throws Exception {
            try (Connection conn = ConnectionFactory.createConnection(conf)) {
                VisibilityClient.clearAuths(conn, new String[] { CONFIDENTIAL }, USER_RW.getShortName());
            } catch (Throwable t) {
                fail("Should not have failed");
            }
            return null;
        }
    });
    authsList = SUPERUSER.runAs(getAuths);
    assertEquals(0, authsList.size());
}