Search in sources :

Example 26 with TBinaryProtocol

use of org.apache.thrift.protocol.TBinaryProtocol in project hbase by apache.

the class DemoClient method run.

private void run() throws Exception {
    TTransport transport = new TSocket(host, port);
    if (secure) {
        Map<String, String> saslProperties = new HashMap<>();
        saslProperties.put(Sasl.QOP, "auth-conf,auth-int,auth");
        /**
           * The Thrift server the DemoClient is trying to connect to
           * must have a matching principal, and support authentication.
           *
           * The HBase cluster must be secure, allow proxy user.
           */
        transport = new TSaslClientTransport("GSSAPI", null, // Thrift server user name, should be an authorized proxy user.
        "hbase", // Thrift server domain
        host, saslProperties, null, transport);
    }
    transport.open();
    TProtocol protocol = new TBinaryProtocol(transport, true, true);
    Hbase.Client client = new Hbase.Client(protocol);
    byte[] t = bytes("demo_table");
    //
    // Scan all tables, look for the demo table and delete it.
    //
    System.out.println("scanning tables...");
    for (ByteBuffer name : client.getTableNames()) {
        System.out.println("  found: " + utf8(name.array()));
        if (utf8(name.array()).equals(utf8(t))) {
            if (client.isTableEnabled(name)) {
                System.out.println("    disabling table: " + utf8(name.array()));
                client.disableTable(name);
            }
            System.out.println("    deleting table: " + utf8(name.array()));
            client.deleteTable(name);
        }
    }
    //
    // Create the demo table with two column families, entry: and unused:
    //
    ArrayList<ColumnDescriptor> columns = new ArrayList<>(2);
    ColumnDescriptor col;
    col = new ColumnDescriptor();
    col.name = ByteBuffer.wrap(bytes("entry:"));
    col.timeToLive = Integer.MAX_VALUE;
    col.maxVersions = 10;
    columns.add(col);
    col = new ColumnDescriptor();
    col.name = ByteBuffer.wrap(bytes("unused:"));
    col.timeToLive = Integer.MAX_VALUE;
    columns.add(col);
    System.out.println("creating table: " + utf8(t));
    try {
        client.createTable(ByteBuffer.wrap(t), columns);
    } catch (AlreadyExists ae) {
        System.out.println("WARN: " + ae.message);
    }
    System.out.println("column families in " + utf8(t) + ": ");
    Map<ByteBuffer, ColumnDescriptor> columnMap = client.getColumnDescriptors(ByteBuffer.wrap(t));
    for (ColumnDescriptor col2 : columnMap.values()) {
        System.out.println("  column: " + utf8(col2.name.array()) + ", maxVer: " + Integer.toString(col2.maxVersions));
    }
    Map<ByteBuffer, ByteBuffer> dummyAttributes = null;
    boolean writeToWal = false;
    //
    // Test UTF-8 handling
    //
    byte[] invalid = { (byte) 'f', (byte) 'o', (byte) 'o', (byte) '-', (byte) 0xfc, (byte) 0xa1, (byte) 0xa1, (byte) 0xa1, (byte) 0xa1 };
    byte[] valid = { (byte) 'f', (byte) 'o', (byte) 'o', (byte) '-', (byte) 0xE7, (byte) 0x94, (byte) 0x9F, (byte) 0xE3, (byte) 0x83, (byte) 0x93, (byte) 0xE3, (byte) 0x83, (byte) 0xBC, (byte) 0xE3, (byte) 0x83, (byte) 0xAB };
    ArrayList<Mutation> mutations;
    // non-utf8 is fine for data
    mutations = new ArrayList<>(1);
    mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(invalid), writeToWal));
    client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("foo")), mutations, dummyAttributes);
    // this row name is valid utf8
    mutations = new ArrayList<>(1);
    mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(valid), writeToWal));
    client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(valid), mutations, dummyAttributes);
    // non-utf8 is now allowed in row names because HBase stores values as binary
    mutations = new ArrayList<>(1);
    mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(invalid), writeToWal));
    client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(invalid), mutations, dummyAttributes);
    // Run a scanner on the rows we just created
    ArrayList<ByteBuffer> columnNames = new ArrayList<>();
    columnNames.add(ByteBuffer.wrap(bytes("entry:")));
    System.out.println("Starting scanner...");
    int scanner = client.scannerOpen(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("")), columnNames, dummyAttributes);
    while (true) {
        List<TRowResult> entry = client.scannerGet(scanner);
        if (entry.isEmpty()) {
            break;
        }
        printRow(entry);
    }
    //
    for (int i = 100; i >= 0; --i) {
        // format row keys as "00000" to "00100"
        NumberFormat nf = NumberFormat.getInstance();
        nf.setMinimumIntegerDigits(5);
        nf.setGroupingUsed(false);
        byte[] row = bytes(nf.format(i));
        mutations = new ArrayList<>(1);
        mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("unused:")), ByteBuffer.wrap(bytes("DELETE_ME")), writeToWal));
        client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, dummyAttributes);
        printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes));
        client.deleteAllRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes);
        // sleep to force later timestamp
        try {
            Thread.sleep(50);
        } catch (InterruptedException e) {
        // no-op
        }
        mutations = new ArrayList<>(2);
        mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:num")), ByteBuffer.wrap(bytes("0")), writeToWal));
        mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:foo")), ByteBuffer.wrap(bytes("FOO")), writeToWal));
        client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, dummyAttributes);
        printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes));
        Mutation m;
        mutations = new ArrayList<>(2);
        m = new Mutation();
        m.column = ByteBuffer.wrap(bytes("entry:foo"));
        m.isDelete = true;
        mutations.add(m);
        m = new Mutation();
        m.column = ByteBuffer.wrap(bytes("entry:num"));
        m.value = ByteBuffer.wrap(bytes("-1"));
        mutations.add(m);
        client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, dummyAttributes);
        printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes));
        mutations = new ArrayList<>();
        mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:num")), ByteBuffer.wrap(bytes(Integer.toString(i))), writeToWal));
        mutations.add(new Mutation(false, ByteBuffer.wrap(bytes("entry:sqr")), ByteBuffer.wrap(bytes(Integer.toString(i * i))), writeToWal));
        client.mutateRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, dummyAttributes);
        printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes));
        // sleep to force later timestamp
        try {
            Thread.sleep(50);
        } catch (InterruptedException e) {
        // no-op
        }
        mutations.clear();
        m = new Mutation();
        m.column = ByteBuffer.wrap(bytes("entry:num"));
        m.value = ByteBuffer.wrap(bytes("-999"));
        mutations.add(m);
        m = new Mutation();
        m.column = ByteBuffer.wrap(bytes("entry:sqr"));
        m.isDelete = true;
        // shouldn't override latest
        client.mutateRowTs(ByteBuffer.wrap(t), ByteBuffer.wrap(row), mutations, 1, dummyAttributes);
        printRow(client.getRow(ByteBuffer.wrap(t), ByteBuffer.wrap(row), dummyAttributes));
        List<TCell> versions = client.getVer(ByteBuffer.wrap(t), ByteBuffer.wrap(row), ByteBuffer.wrap(bytes("entry:num")), 10, dummyAttributes);
        printVersions(ByteBuffer.wrap(row), versions);
        if (versions.isEmpty()) {
            System.out.println("FATAL: wrong # of versions");
            System.exit(-1);
        }
        List<TCell> result = client.get(ByteBuffer.wrap(t), ByteBuffer.wrap(row), ByteBuffer.wrap(bytes("entry:foo")), dummyAttributes);
        if (!result.isEmpty()) {
            System.out.println("FATAL: shouldn't get here");
            System.exit(-1);
        }
        System.out.println("");
    }
    // scan all rows/columnNames
    columnNames.clear();
    for (ColumnDescriptor col2 : client.getColumnDescriptors(ByteBuffer.wrap(t)).values()) {
        System.out.println("column with name: " + new String(col2.name.array()));
        System.out.println(col2.toString());
        columnNames.add(col2.name);
    }
    System.out.println("Starting scanner...");
    scanner = client.scannerOpenWithStop(ByteBuffer.wrap(t), ByteBuffer.wrap(bytes("00020")), ByteBuffer.wrap(bytes("00040")), columnNames, dummyAttributes);
    while (true) {
        List<TRowResult> entry = client.scannerGet(scanner);
        if (entry.isEmpty()) {
            System.out.println("Scanner finished");
            break;
        }
        printRow(entry);
    }
    transport.close();
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TSaslClientTransport(org.apache.thrift.transport.TSaslClientTransport) TProtocol(org.apache.thrift.protocol.TProtocol) TSocket(org.apache.thrift.transport.TSocket) ColumnDescriptor(org.apache.hadoop.hbase.thrift.generated.ColumnDescriptor) TCell(org.apache.hadoop.hbase.thrift.generated.TCell) ByteBuffer(java.nio.ByteBuffer) TBinaryProtocol(org.apache.thrift.protocol.TBinaryProtocol) TTransport(org.apache.thrift.transport.TTransport) Mutation(org.apache.hadoop.hbase.thrift.generated.Mutation) AlreadyExists(org.apache.hadoop.hbase.thrift.generated.AlreadyExists) TRowResult(org.apache.hadoop.hbase.thrift.generated.TRowResult) Hbase(org.apache.hadoop.hbase.thrift.generated.Hbase) NumberFormat(java.text.NumberFormat)

Example 27 with TBinaryProtocol

use of org.apache.thrift.protocol.TBinaryProtocol in project hbase by apache.

the class DemoClient method run.

public void run() throws Exception {
    int timeout = 10000;
    boolean framed = false;
    TTransport transport = new TSocket(host, port, timeout);
    if (framed) {
        transport = new TFramedTransport(transport);
    } else if (secure) {
        /**
       * The Thrift server the DemoClient is trying to connect to
       * must have a matching principal, and support authentication.
       *
       * The HBase cluster must be secure, allow proxy user.
       */
        Map<String, String> saslProperties = new HashMap<>();
        saslProperties.put(Sasl.QOP, "auth-conf,auth-int,auth");
        transport = new TSaslClientTransport("GSSAPI", null, // Thrift server user name, should be an authorized proxy user
        user != null ? user : "hbase", // Thrift server domain
        host, saslProperties, null, transport);
    }
    TProtocol protocol = new TBinaryProtocol(transport);
    // This is our thrift client.
    THBaseService.Iface client = new THBaseService.Client(protocol);
    // open the transport
    transport.open();
    ByteBuffer table = ByteBuffer.wrap("example".getBytes());
    TPut put = new TPut();
    put.setRow("row1".getBytes());
    TColumnValue columnValue = new TColumnValue();
    columnValue.setFamily("family1".getBytes());
    columnValue.setQualifier("qualifier1".getBytes());
    columnValue.setValue("value1".getBytes());
    List<TColumnValue> columnValues = new ArrayList<>(1);
    columnValues.add(columnValue);
    put.setColumnValues(columnValues);
    client.put(table, put);
    TGet get = new TGet();
    get.setRow("row1".getBytes());
    TResult result = client.get(table, get);
    System.out.print("row = " + new String(result.getRow()));
    for (TColumnValue resultColumnValue : result.getColumnValues()) {
        System.out.print("family = " + new String(resultColumnValue.getFamily()));
        System.out.print("qualifier = " + new String(resultColumnValue.getFamily()));
        System.out.print("value = " + new String(resultColumnValue.getValue()));
        System.out.print("timestamp = " + resultColumnValue.getTimestamp());
    }
    transport.close();
}
Also used : TGet(org.apache.hadoop.hbase.thrift2.generated.TGet) ArrayList(java.util.ArrayList) TSaslClientTransport(org.apache.thrift.transport.TSaslClientTransport) TColumnValue(org.apache.hadoop.hbase.thrift2.generated.TColumnValue) ByteBuffer(java.nio.ByteBuffer) TResult(org.apache.hadoop.hbase.thrift2.generated.TResult) TBinaryProtocol(org.apache.thrift.protocol.TBinaryProtocol) TProtocol(org.apache.thrift.protocol.TProtocol) TFramedTransport(org.apache.thrift.transport.TFramedTransport) TTransport(org.apache.thrift.transport.TTransport) TPut(org.apache.hadoop.hbase.thrift2.generated.TPut) HashMap(java.util.HashMap) Map(java.util.Map) TSocket(org.apache.thrift.transport.TSocket) THBaseService(org.apache.hadoop.hbase.thrift2.generated.THBaseService)

Example 28 with TBinaryProtocol

use of org.apache.thrift.protocol.TBinaryProtocol in project storm by apache.

the class ThriftClient method reconnect.

public synchronized void reconnect() {
    close();
    TSocket socket = null;
    try {
        socket = new TSocket(_host, _port);
        if (_timeout != null) {
            socket.setTimeout(_timeout);
        }
        //locate login configuration 
        Configuration login_conf = AuthUtils.GetConfiguration(_conf);
        //construct a transport plugin
        ITransportPlugin transportPlugin = AuthUtils.GetTransportPlugin(_type, _conf, login_conf);
        //TODO get this from type instead of hardcoding to Nimbus.
        //establish client-server transport via plugin
        //do retries if the connect fails
        TBackoffConnect connectionRetry = new TBackoffConnect(Utils.getInt(_conf.get(Config.STORM_NIMBUS_RETRY_TIMES)), Utils.getInt(_conf.get(Config.STORM_NIMBUS_RETRY_INTERVAL)), Utils.getInt(_conf.get(Config.STORM_NIMBUS_RETRY_INTERVAL_CEILING)), _retryForever);
        _transport = connectionRetry.doConnectWithRetry(transportPlugin, socket, _host, _asUser);
    } catch (Exception ex) {
        // close the socket, which releases connection if it has created any.
        if (socket != null) {
            try {
                socket.close();
            } catch (Exception e) {
            }
        }
        throw new RuntimeException(ex);
    }
    _protocol = null;
    if (_transport != null) {
        _protocol = new TBinaryProtocol(_transport);
    }
}
Also used : Configuration(javax.security.auth.login.Configuration) TBinaryProtocol(org.apache.thrift.protocol.TBinaryProtocol) IOException(java.io.IOException) TSocket(org.apache.thrift.transport.TSocket)

Example 29 with TBinaryProtocol

use of org.apache.thrift.protocol.TBinaryProtocol in project whirr by apache.

the class CassandraServiceTest method waitForCassandra.

private void waitForCassandra() {
    for (Instance instance : cluster.getInstances()) {
        while (true) {
            try {
                TSocket socket = new TSocket(instance.getPublicAddress().getHostAddress(), CassandraService.CLIENT_PORT);
                socket.open();
                TBinaryProtocol protocol = new TBinaryProtocol(socket);
                Cassandra.Client client = new Cassandra.Client(protocol);
                client.describe_cluster_name();
                socket.close();
                break;
            } catch (TException e) {
                System.out.print(".");
                try {
                    Thread.sleep(1000);
                } catch (InterruptedException e1) {
                    break;
                }
            }
        }
    }
}
Also used : TException(org.apache.thrift.TException) TBinaryProtocol(org.apache.thrift.protocol.TBinaryProtocol) Instance(org.apache.whirr.service.Cluster.Instance) Cassandra(org.apache.cassandra.thrift.Cassandra) TSocket(org.apache.thrift.transport.TSocket)

Example 30 with TBinaryProtocol

use of org.apache.thrift.protocol.TBinaryProtocol in project whirr by apache.

the class CassandraServiceTest method testInstances.

@Test
public void testInstances() throws Exception {
    Set<String> endPoints = new HashSet<String>();
    for (Instance instance : cluster.getInstances()) {
        TSocket socket = new TSocket(instance.getPublicAddress().getHostAddress(), CassandraService.CLIENT_PORT);
        socket.open();
        TBinaryProtocol protocol = new TBinaryProtocol(socket);
        Cassandra.Client client = new Cassandra.Client(protocol);
        List<TokenRange> tr = client.describe_ring(KEYSPACE);
        for (TokenRange tokenRange : tr) {
            endPoints.addAll(tokenRange.endpoints);
        }
        socket.close();
    }
    for (Instance instance : cluster.getInstances()) {
        String address = instance.getPrivateAddress().getHostAddress();
        assertTrue(address + " not in cluster!", endPoints.remove(address));
    }
    assertTrue("Unknown node returned: " + endPoints.toString(), endPoints.isEmpty());
}
Also used : TBinaryProtocol(org.apache.thrift.protocol.TBinaryProtocol) Instance(org.apache.whirr.service.Cluster.Instance) Cassandra(org.apache.cassandra.thrift.Cassandra) TokenRange(org.apache.cassandra.thrift.TokenRange) HashSet(java.util.HashSet) TSocket(org.apache.thrift.transport.TSocket) Test(org.junit.Test)

Aggregations

TBinaryProtocol (org.apache.thrift.protocol.TBinaryProtocol)52 TSocket (org.apache.thrift.transport.TSocket)34 TProtocol (org.apache.thrift.protocol.TProtocol)26 TTransport (org.apache.thrift.transport.TTransport)25 TFramedTransport (org.apache.thrift.transport.TFramedTransport)21 TException (org.apache.thrift.TException)16 TIOStreamTransport (org.apache.thrift.transport.TIOStreamTransport)11 IOException (java.io.IOException)10 TMessage (org.apache.thrift.protocol.TMessage)8 TTransportException (org.apache.thrift.transport.TTransportException)8 Test (org.junit.Test)7 Request (com.alibaba.dubbo.remoting.exchange.Request)6 Demo (com.alibaba.dubbo.rpc.gen.thrift.Demo)6 ChannelBuffer (com.alibaba.dubbo.remoting.buffer.ChannelBuffer)5 RpcResult (com.alibaba.dubbo.rpc.RpcResult)5 RandomAccessByteArrayOutputStream (com.alibaba.dubbo.rpc.protocol.thrift.io.RandomAccessByteArrayOutputStream)5 Hbase (org.apache.hadoop.hbase.thrift.generated.Hbase)5 URL (com.alibaba.dubbo.common.URL)4 Channel (com.alibaba.dubbo.remoting.Channel)4 Response (com.alibaba.dubbo.remoting.exchange.Response)4