Examples with TConfiguration org.apache.thrift.TConfiguration used on opensource projects

Example 1 with TConfiguration

use of org.apache.thrift.TConfiguration in project hive by apache.

the class HiveMetaStoreClientPreCatalog method open.

private void open() throws MetaException {
    isConnected = false;
    TTransportException tte = null;
    boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
    boolean useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
    boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
    boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
    int clientSocketTimeout = (int) MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
    for (int attempt = 0; !isConnected && attempt < retries; ++attempt) {
        for (URI store : metastoreUris) {
            LOG.info("Trying to connect to metastore with URI " + store);
            try {
                if (useSSL) {
                    try {
                        String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();
                        if (trustStorePath.isEmpty()) {
                            throw new IllegalArgumentException(ConfVars.SSL_TRUSTSTORE_PATH.toString() + " Not configured for SSL connection");
                        }
                        String trustStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
                        String trustStoreType = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
                        String trustStoreAlgorithm = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
                        // Create an SSL socket and connect
                        transport = SecurityUtils.getSSLSocket(store.getHost(), store.getPort(), clientSocketTimeout, trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm);
                        LOG.info("Opened an SSL connection to metastore, current connections: " + connCount.incrementAndGet());
                    } catch (IOException e) {
                        throw new IllegalArgumentException(e);
                    } catch (TTransportException e) {
                        tte = e;
                        throw new MetaException(e.toString());
                    }
                } else {
                    try {
                        transport = new TSocket(new TConfiguration(), store.getHost(), store.getPort(), clientSocketTimeout);
                    } catch (TTransportException e) {
                        tte = e;
                        throw new MetaException(e.toString());
                    }
                }
                if (useSasl) {
                    // Wrap thrift connection with SASL for secure connection.
                    try {
                        HadoopThriftAuthBridge.Client authBridge = HadoopThriftAuthBridge.getBridge().createClient();
                        // check if we should use delegation tokens to authenticate
                        // the call below gets hold of the tokens if they are set up by hadoop
                        // this should happen on the map/reduce tasks if the client added the
                        // tokens into hadoop's credential store in the front end during job
                        // submission.
                        String tokenSig = MetastoreConf.getVar(conf, ConfVars.TOKEN_SIGNATURE);
                        // tokenSig could be null
                        tokenStrForm = SecurityUtils.getTokenStrForm(tokenSig);
                        if (tokenStrForm != null) {
                            LOG.info("HMSC::open(): Found delegation token. Creating DIGEST-based thrift connection.");
                            // authenticate using delegation tokens via the "DIGEST" mechanism
                            transport = authBridge.createClientTransport(null, store.getHost(), "DIGEST", tokenStrForm, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
                        } else {
                            LOG.info("HMSC::open(): Could not find delegation token. Creating KERBEROS-based thrift connection.");
                            String principalConfig = MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL);
                            transport = authBridge.createClientTransport(principalConfig, store.getHost(), "KERBEROS", null, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
                        }
                    } catch (IOException ioe) {
                        LOG.error("Couldn't create client transport", ioe);
                        throw new MetaException(ioe.toString());
                    }
                } else {
                    if (useFramedTransport) {
                        try {
                            transport = new TFramedTransport(transport);
                        } catch (TTransportException e) {
                            LOG.error("Couldn't create client transport", e);
                            throw new MetaException(e.toString());
                        }
                    }
                }
                final TProtocol protocol;
                if (useCompactProtocol) {
                    protocol = new TCompactProtocol(transport);
                } else {
                    protocol = new TBinaryProtocol(transport);
                }
                client = new ThriftHiveMetastore.Client(protocol);
                try {
                    if (!transport.isOpen()) {
                        transport.open();
                        LOG.info("Opened a connection to metastore, current connections: " + connCount.incrementAndGet());
                    }
                    isConnected = true;
                } catch (TTransportException e) {
                    tte = e;
                    if (LOG.isDebugEnabled()) {
                        LOG.warn("Failed to connect to the MetaStore Server...", e);
                    } else {
                        // Don't print full exception trace if DEBUG is not on.
                        LOG.warn("Failed to connect to the MetaStore Server...");
                    }
                }
                if (isConnected && !useSasl && MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) {
                    // Call set_ugi, only in unsecure mode.
                    try {
                        UserGroupInformation ugi = SecurityUtils.getUGI();
                        client.set_ugi(ugi.getUserName(), Arrays.asList(ugi.getGroupNames()));
                    } catch (LoginException e) {
                        LOG.warn("Failed to do login. set_ugi() is not successful, " + "Continuing without it.", e);
                    } catch (IOException e) {
                        LOG.warn("Failed to find ugi of client set_ugi() is not successful, " + "Continuing without it.", e);
                    } catch (TException e) {
                        LOG.warn("set_ugi() not successful, Likely cause: new client talking to old server. " + "Continuing without it.", e);
                    }
                }
            } catch (MetaException e) {
                LOG.error("Unable to connect to metastore with URI " + store + " in attempt " + attempt, e);
            }
            if (isConnected) {
                break;
            }
        }
        // Wait before launching the next round of connection retries.
        if (!isConnected && retryDelaySeconds > 0) {
            try {
                LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt.");
                Thread.sleep(retryDelaySeconds * 1000);
            } catch (InterruptedException ignore) {
            }
        }
    }
    if (!isConnected) {
        throw new MetaException("Could not connect to meta store using any of the URIs provided." + " Most recent failure: " + StringUtils.stringifyException(tte));
    }
    snapshotActiveConf();
    LOG.info("Connected to metastore.");
}

Example 2 with TConfiguration

use of org.apache.thrift.TConfiguration in project hive by apache.

the class TestTCTLSeparatedProtocol method testShouldThrowRunTimeExceptionIfUnableToInitializeTokenizer.

@Test
public void testShouldThrowRunTimeExceptionIfUnableToInitializeTokenizer() throws Exception {
    TCTLSeparatedProtocol separatedProtocol = new TCTLSeparatedProtocol(new TTransport() {

        @Override
        public void close() {
        }

        @Override
        public boolean isOpen() {
            return false;
        }

        @Override
        public void open() throws TTransportException {
        }

        @Override
        public int read(byte[] buf, int off, int len) throws TTransportException {
            throw new TTransportException();
        }

        @Override
        public void write(byte[] buf, int off, int len) throws TTransportException {
        }

        @Override
        public TConfiguration getConfiguration() {
            return null;
        }

        @Override
        public void updateKnownMessageSize(long l) throws TTransportException {
        }

        @Override
        public void checkReadBytesAvailable(long l) throws TTransportException {
        }
    });
    separatedProtocol.initialize(null, new Properties());
    try {
        separatedProtocol.readStructBegin();
        fail("Runtime Exception is expected if the initialization of tokenizer failed.");
    } catch (Exception e) {
        assertTrue(e.getCause() instanceof TTransportException);
    }
}

Example 3 with TConfiguration

use of org.apache.thrift.TConfiguration in project hbase by apache.

the class DemoClient method run.

public void run() throws Exception {
    int timeout = 10000;
    boolean framed = false;
    TTransport transport = new TSocket(new TConfiguration(), host, port, timeout);
    if (framed) {
        transport = new TFramedTransport(transport);
    } else if (secure) {
        /*
       * The Thrift server the DemoClient is trying to connect to
       * must have a matching principal, and support authentication.
       *
       * The HBase cluster must be secure, allow proxy user.
       */
        Map<String, String> saslProperties = new HashMap<>();
        saslProperties.put(Sasl.QOP, "auth-conf,auth-int,auth");
        transport = new TSaslClientTransport("GSSAPI", null, // Thrift server user name, should be an authorized proxy user
        user != null ? user : "hbase", // Thrift server domain
        host, saslProperties, null, transport);
    }
    TProtocol protocol = new TBinaryProtocol(transport);
    // This is our thrift client.
    THBaseService.Iface client = new THBaseService.Client(protocol);
    // open the transport
    transport.open();
    ByteBuffer table = ByteBuffer.wrap(Bytes.toBytes("example"));
    TPut put = new TPut();
    put.setRow(Bytes.toBytes("row1"));
    TColumnValue columnValue = new TColumnValue();
    columnValue.setFamily(Bytes.toBytes("family1"));
    columnValue.setQualifier(Bytes.toBytes("qualifier1"));
    columnValue.setValue(Bytes.toBytes("value1"));
    List<TColumnValue> columnValues = new ArrayList<>(1);
    columnValues.add(columnValue);
    put.setColumnValues(columnValues);
    client.put(table, put);
    TGet get = new TGet();
    get.setRow(Bytes.toBytes("row1"));
    TResult result = client.get(table, get);
    System.out.print("row = " + new String(result.getRow()));
    for (TColumnValue resultColumnValue : result.getColumnValues()) {
        System.out.print("family = " + new String(resultColumnValue.getFamily()));
        System.out.print("qualifier = " + new String(resultColumnValue.getFamily()));
        System.out.print("value = " + new String(resultColumnValue.getValue()));
        System.out.print("timestamp = " + resultColumnValue.getTimestamp());
    }
    transport.close();
}

Example 4 with TConfiguration

use of org.apache.thrift.TConfiguration in project hive by apache.

the class HiveMetaStoreClient method open.

private void open() throws MetaException {
    isConnected = false;
    TTransportException tte = null;
    MetaException recentME = null;
    boolean useSSL = MetastoreConf.getBoolVar(conf, ConfVars.USE_SSL);
    boolean useSasl = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_SASL);
    String clientAuthMode = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_AUTH_MODE);
    boolean usePasswordAuth = false;
    boolean useFramedTransport = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
    boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
    int clientSocketTimeout = (int) MetastoreConf.getTimeVar(conf, ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
    if (clientAuthMode != null) {
        usePasswordAuth = "PLAIN".equalsIgnoreCase(clientAuthMode);
    }
    for (int attempt = 0; !isConnected && attempt < retries; ++attempt) {
        for (URI store : metastoreUris) {
            LOG.info("Trying to connect to metastore with URI ({})", store);
            try {
                if (useSSL) {
                    try {
                        String trustStorePath = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_PATH).trim();
                        if (trustStorePath.isEmpty()) {
                            throw new IllegalArgumentException(ConfVars.SSL_TRUSTSTORE_PATH + " Not configured for SSL connection");
                        }
                        String trustStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
                        String trustStoreType = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTSTORE_TYPE).trim();
                        String trustStoreAlgorithm = MetastoreConf.getVar(conf, ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
                        // Create an SSL socket and connect
                        transport = SecurityUtils.getSSLSocket(store.getHost(), store.getPort(), clientSocketTimeout, trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm);
                        final int newCount = connCount.incrementAndGet();
                        LOG.debug("Opened an SSL connection to metastore, current connections: {}", newCount);
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("METASTORE SSL CONNECTION TRACE - open [{}]", System.identityHashCode(this), new Exception());
                        }
                    } catch (IOException e) {
                        throw new IllegalArgumentException(e);
                    } catch (TTransportException e) {
                        tte = e;
                        throw new MetaException(e.toString());
                    }
                } else {
                    try {
                        transport = new TSocket(new TConfiguration(), store.getHost(), store.getPort(), clientSocketTimeout);
                    } catch (TTransportException e) {
                        tte = e;
                        throw new MetaException(e.toString());
                    }
                }
                if (usePasswordAuth) {
                    // we are using PLAIN Sasl connection with user/password
                    LOG.debug("HMSC::open(): Creating plain authentication thrift connection.");
                    String userName = MetastoreConf.getVar(conf, ConfVars.METASTORE_CLIENT_PLAIN_USERNAME);
                    if (null == userName || userName.isEmpty()) {
                        throw new MetaException("No user specified for plain transport.");
                    }
                    // by configuration "hadoop.security.credential.provider.path".
                    try {
                        String passwd = null;
                        char[] pwdCharArray = conf.getPassword(userName);
                        if (null != pwdCharArray) {
                            passwd = new String(pwdCharArray);
                        }
                        if (null == passwd) {
                            throw new MetaException("No password found for user " + userName);
                        }
                        // Overlay the SASL transport on top of the base socket transport (SSL or non-SSL)
                        transport = MetaStorePlainSaslHelper.getPlainTransport(userName, passwd, transport);
                    } catch (IOException | TTransportException sasle) {
                        // IOException covers SaslException
                        LOG.error("Could not create client transport", sasle);
                        throw new MetaException(sasle.toString());
                    }
                } else if (useSasl) {
                    // Wrap thrift connection with SASL for secure connection.
                    try {
                        HadoopThriftAuthBridge.Client authBridge = HadoopThriftAuthBridge.getBridge().createClient();
                        // check if we should use delegation tokens to authenticate
                        // the call below gets hold of the tokens if they are set up by hadoop
                        // this should happen on the map/reduce tasks if the client added the
                        // tokens into hadoop's credential store in the front end during job
                        // submission.
                        String tokenSig = MetastoreConf.getVar(conf, ConfVars.TOKEN_SIGNATURE);
                        // tokenSig could be null
                        tokenStrForm = SecurityUtils.getTokenStrForm(tokenSig);
                        if (tokenStrForm != null) {
                            LOG.debug("HMSC::open(): Found delegation token. Creating DIGEST-based thrift connection.");
                            // authenticate using delegation tokens via the "DIGEST" mechanism
                            transport = authBridge.createClientTransport(null, store.getHost(), "DIGEST", tokenStrForm, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
                        } else {
                            LOG.debug("HMSC::open(): Could not find delegation token. Creating KERBEROS-based thrift connection.");
                            String principalConfig = MetastoreConf.getVar(conf, ConfVars.KERBEROS_PRINCIPAL);
                            transport = authBridge.createClientTransport(principalConfig, store.getHost(), "KERBEROS", null, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
                        }
                    } catch (IOException ioe) {
                        LOG.error("Failed to create client transport", ioe);
                        throw new MetaException(ioe.toString());
                    }
                } else {
                    if (useFramedTransport) {
                        try {
                            transport = new TFramedTransport(transport);
                        } catch (TTransportException e) {
                            LOG.error("Failed to create client transport", e);
                            throw new MetaException(e.toString());
                        }
                    }
                }
                final TProtocol protocol;
                if (useCompactProtocol) {
                    protocol = new TCompactProtocol(transport);
                } else {
                    protocol = new TBinaryProtocol(transport);
                }
                client = new ThriftHiveMetastore.Client(protocol);
                try {
                    if (!transport.isOpen()) {
                        transport.open();
                        final int newCount = connCount.incrementAndGet();
                        LOG.info("Opened a connection to metastore, URI ({}) " + "current connections: {}", store, newCount);
                        if (LOG.isTraceEnabled()) {
                            LOG.trace("METASTORE CONNECTION TRACE - open [{}]", System.identityHashCode(this), new Exception());
                        }
                    }
                    isConnected = true;
                } catch (TTransportException e) {
                    tte = e;
                    LOG.warn("Failed to connect to the MetaStore Server URI ({})", store);
                    LOG.debug("Failed to connect to the MetaStore Server URI ({})", store, e);
                }
                if (isConnected && !useSasl && !usePasswordAuth && MetastoreConf.getBoolVar(conf, ConfVars.EXECUTE_SET_UGI)) {
                    // Call set_ugi, only in unsecure mode.
                    try {
                        UserGroupInformation ugi = SecurityUtils.getUGI();
                        client.set_ugi(ugi.getUserName(), Arrays.asList(ugi.getGroupNames()));
                    } catch (LoginException e) {
                        LOG.warn("Failed to do login. set_ugi() is not successful, " + "Continuing without it.", e);
                    } catch (IOException e) {
                        LOG.warn("Failed to find ugi of client set_ugi() is not successful, " + "Continuing without it.", e);
                    } catch (TException e) {
                        LOG.warn("set_ugi() not successful, Likely cause: new client talking to old server. " + "Continuing without it.", e);
                    }
                }
            } catch (MetaException e) {
                recentME = e;
                LOG.error("Failed to connect to metastore with URI (" + store + ") in attempt " + attempt, e);
            }
            if (isConnected) {
                break;
            }
        }
        // Wait before launching the next round of connection retries.
        if (!isConnected && retryDelaySeconds > 0) {
            try {
                LOG.info("Waiting " + retryDelaySeconds + " seconds before next connection attempt.");
                Thread.sleep(retryDelaySeconds * 1000);
            } catch (InterruptedException ignore) {
            }
        }
    }
    if (!isConnected) {
        // Either tte or recentME should be set but protect from a bug which causes both of them to
        // be null. When MetaException wraps TTransportException, tte will be set so stringify that
        // directly.
        String exceptionString = "Unknown exception";
        if (tte != null) {
            exceptionString = StringUtils.stringifyException(tte);
        } else if (recentME != null) {
            exceptionString = StringUtils.stringifyException(recentME);
        }
        throw new MetaException("Could not connect to meta store using any of the URIs provided." + " Most recent failure: " + exceptionString);
    }
    snapshotActiveConf();
}

Example 5 with TConfiguration

use of org.apache.thrift.TConfiguration in project hive by apache.

the class HMSClient method open.

private TTransport open(Configuration conf, @NotNull URI uri) throws TException, IOException, LoginException {
    boolean useSSL = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_SSL);
    boolean useSasl = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_THRIFT_SASL);
    boolean useFramedTransport = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_THRIFT_FRAMED_TRANSPORT);
    boolean useCompactProtocol = MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.USE_THRIFT_COMPACT_PROTOCOL);
    int clientSocketTimeout = (int) MetastoreConf.getTimeVar(conf, MetastoreConf.ConfVars.CLIENT_SOCKET_TIMEOUT, TimeUnit.MILLISECONDS);
    LOG.debug("Connecting to {}, framedTransport = {}", uri, useFramedTransport);
    String host = uri.getHost();
    int port = uri.getPort();
    // Sasl/SSL code is copied from HiveMetastoreCLient
    if (!useSSL) {
        transport = new TSocket(new TConfiguration(), host, port, clientSocketTimeout);
    } else {
        String trustStorePath = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PATH).trim();
        if (trustStorePath.isEmpty()) {
            throw new IllegalArgumentException(MetastoreConf.ConfVars.SSL_TRUSTSTORE_PATH.toString() + " Not configured for SSL connection");
        }
        String trustStorePassword = MetastoreConf.getPassword(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_PASSWORD);
        String trustStoreType = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_TRUSTSTORE_TYPE).trim();
        String trustStoreAlgorithm = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.SSL_TRUSTMANAGERFACTORY_ALGORITHM).trim();
        // Create an SSL socket and connect
        transport = SecurityUtils.getSSLSocket(host, port, clientSocketTimeout, trustStorePath, trustStorePassword, trustStoreType, trustStoreAlgorithm);
        LOG.info("Opened an SSL connection to metastore, current connections");
    }
    if (useSasl) {
        // Wrap thrift connection with SASL for secure connection.
        HadoopThriftAuthBridge.Client authBridge = HadoopThriftAuthBridge.getBridge().createClient();
        // check if we should use delegation tokens to authenticate
        // the call below gets hold of the tokens if they are set up by hadoop
        // this should happen on the map/reduce tasks if the client added the
        // tokens into hadoop's credential store in the front end during job
        // submission.
        String tokenSig = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.TOKEN_SIGNATURE);
        // tokenSig could be null
        String tokenStrForm = SecurityUtils.getTokenStrForm(tokenSig);
        if (tokenStrForm != null) {
            LOG.info("HMSC::open(): Found delegation token. Creating DIGEST-based thrift connection.");
            // authenticate using delegation tokens via the "DIGEST" mechanism
            transport = authBridge.createClientTransport(null, host, "DIGEST", tokenStrForm, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
        } else {
            LOG.info("HMSC::open(): Could not find delegation token. Creating KERBEROS-based thrift connection.");
            String principalConfig = MetastoreConf.getVar(conf, MetastoreConf.ConfVars.KERBEROS_PRINCIPAL);
            transport = authBridge.createClientTransport(principalConfig, host, "KERBEROS", null, transport, MetaStoreUtils.getMetaStoreSaslProperties(conf, useSSL));
        }
    } else {
        if (useFramedTransport) {
            transport = new TFramedTransport(transport);
        }
    }
    final TProtocol protocol;
    if (useCompactProtocol) {
        protocol = new TCompactProtocol(transport);
    } else {
        protocol = new TBinaryProtocol(transport);
    }
    client = new ThriftHiveMetastore.Client(protocol);
    if (!transport.isOpen()) {
        transport.open();
        LOG.info("Opened a connection to metastore, current connections");
        if (!useSasl && MetastoreConf.getBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI)) {
            // Call set_ugi, only in unsecure mode.
            try {
                UserGroupInformation ugi = SecurityUtils.getUGI();
                client.set_ugi(ugi.getUserName(), Arrays.asList(ugi.getGroupNames()));
            } catch (LoginException e) {
                LOG.warn("Failed to do login. set_ugi() is not successful, " + "Continuing without it.", e);
            } catch (IOException e) {
                LOG.warn("Failed to find ugi of client set_ugi() is not successful, " + "Continuing without it.", e);
            } catch (TException e) {
                LOG.warn("set_ugi() not successful, Likely cause: new client talking to old server. " + "Continuing without it.", e);
            }
        }
    }
    LOG.debug("Connected to metastore, using compact protocol = {}", useCompactProtocol);
    return transport;
}