Examples with ClientConfig org.voltdb.client.ClientConfig used on opensource projects

Example 11 with ClientConfig

use of org.voltdb.client.ClientConfig in project voltdb by VoltDB.

the class RegressionSuite method getFullyConnectedClient.

public Client getFullyConnectedClient(long timeout) throws IOException {
    final List<String> listeners = m_config.getListenerAddresses();
    final Random r = new Random();
    ClientConfig config = new ClientConfigForTest(m_username, m_password);
    config.setConnectionResponseTimeout(timeout);
    config.setProcedureCallTimeout(timeout);
    final Client client = ClientFactory.createClient(config);
    for (String listener : listeners) {
        // Use the port generated by LocalCluster if applicable
        try {
            client.createConnection(listener);
        }// retry once
         catch (ConnectException e) {
            listener = listeners.get(r.nextInt(listeners.size()));
            client.createConnection(listener);
        }
    }
    m_clients.add(client);
    return client;
}

Example 12 with ClientConfig

use of org.voltdb.client.ClientConfig in project voltdb by VoltDB.

the class KafkaLoader10 method processKafkaMessages.

private void processKafkaMessages() throws Exception {
    // Split server list
    final String[] serverlist = m_cliOptions.servers.split(",");
    // If we need to prompt the user for a VoltDB password, do so.
    m_cliOptions.password = CLIConfig.readPasswordIfNeeded(m_cliOptions.user, m_cliOptions.password, "Enter password: ");
    // Create connection
    final ClientConfig clientConfig = new ClientConfig(m_cliOptions.user, m_cliOptions.password, null);
    if (m_cliOptions.ssl != null && !m_cliOptions.ssl.trim().isEmpty()) {
        clientConfig.setTrustStoreConfigFromPropertyFile(m_cliOptions.ssl);
        clientConfig.enableSSL();
    }
    clientConfig.setProcedureCallTimeout(0);
    m_client = getClient(clientConfig, serverlist);
    if (m_cliOptions.useSuppliedProcedure) {
        m_loader = new CSVTupleDataLoader((ClientImpl) m_client, m_cliOptions.procedure, new KafkaBulkLoaderCallback());
    } else {
        m_loader = new CSVBulkDataLoader((ClientImpl) m_client, m_cliOptions.table, m_cliOptions.batch, m_cliOptions.update, new KafkaBulkLoaderCallback());
    }
    m_loader.setFlushInterval(m_cliOptions.flush, m_cliOptions.flush);
    if ((m_executorService = getExecutor()) != null) {
        if (m_cliOptions.useSuppliedProcedure) {
            m_log.info("Kafka Consumer from topic: " + m_cliOptions.topic + " Started using procedure: " + m_cliOptions.procedure);
        } else {
            m_log.info("Kafka Consumer from topic: " + m_cliOptions.topic + " Started for table: " + m_cliOptions.table);
        }
        m_executorService.shutdown();
        m_executorService.awaitTermination(365, TimeUnit.DAYS);
        m_executorService = null;
    }
}

Example 13 with ClientConfig

use of org.voltdb.client.ClientConfig in project voltdb by VoltDB.

the class LatencyLogger method main.

public static void main(String[] args) throws Exception {
    final String hostname;
    if (args.length == 1) {
        readHistogramFromFile(args[0]);
        return;
    }
    if (args.length != 3) {
        if (args.length == 4) {
            hostname = args[3];
        } else {
            System.out.println(usage());
            return;
        }
    } else {
        hostname = args[0];
    }
    final String server = args[0];
    int dur = 0;
    try {
        dur = Integer.valueOf(args[2]);
        if (dur < 1) {
            throw new NumberFormatException();
        }
    } catch (NumberFormatException e) {
        System.out.println("reportIntervalSeconds should be greater than or equal to 1");
        System.out.println(usage());
        System.exit(0);
    }
    final int duration = dur;
    // start with an empty password
    String username = "";
    String password = "";
    // if credentials set in env, use them
    if (System.getenv().containsKey("VOLTDBUSER")) {
        username = System.getenv("VOLTDBUSER");
    }
    if (System.getenv().containsKey("VOLTDBPASSWORD")) {
        password = System.getenv("VOLTDBPASSWORD");
    }
    // create the client with our credentials
    ClientConfig clientConfig = new ClientConfig(username, password);
    final Client c = ClientFactory.createClient(clientConfig);
    int port = 0;
    try {
        port = Integer.valueOf(args[1]);
    } catch (NumberFormatException e) {
        System.out.println("Failed to parse port number.");
        System.out.println("Usage server port reportIntervalSeconds");
        System.exit(0);
    }
    System.out.println("Connecting to " + server + " port " + port);
    c.createConnection(args[0], port);
    System.out.printf("%12s, %10s, %10s, %10s, %10s, %10s, %10s, %10s\n", "TIMESTAMP", "COUNT", "TPS", "95", "99", "99.9", "99.99", "99.999");
    final SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss.SSS");
    ScheduledExecutorService ses = Executors.newSingleThreadScheduledExecutor();
    ses.scheduleAtFixedRate(new Runnable() {

        @Override
        public void run() {
            VoltTable table = null;
            try {
                table = c.callProcedure("@Statistics", "LATENCY_HISTOGRAM", 0).getResults()[0];
            } catch (IOException | ProcCallException e) {
                System.out.println("Failed to get statistics:");
                e.printStackTrace();
                System.exit(0);
            }
            List<String> hostnames = new ArrayList<String>();
            String tableHostname = "";
            while (!hostname.equalsIgnoreCase(tableHostname)) {
                if (!table.advanceRow()) {
                    System.out.println("Server host name " + server + " not found. Valid host names are " + hostnames.toString());
                    System.exit(0);
                }
                tableHostname = table.getString(2);
                hostnames.add(tableHostname);
            }
            Date now = new Date(table.getLong(0));
            Histogram newHistogram = AbstractHistogram.fromCompressedBytes(table.getVarbinary(4), CompressionStrategySnappy.INSTANCE);
            Histogram diffHistogram;
            if (m_histogramData == null) {
                diffHistogram = newHistogram;
            } else {
                diffHistogram = Histogram.diff(newHistogram, m_histogramData);
            }
            long totalCount = diffHistogram.getTotalCount();
            if (totalCount > 0) {
                System.out.printf("%12s, %10d, %10.0f, %8.2fms, %8.2fms, %8.2fms, %8.2fms, %8.2fms\n", sdf.format(now), totalCount, ((double) totalCount / duration), (diffHistogram.getValueAtPercentile(95.0D) / 1000.0D), (diffHistogram.getValueAtPercentile(99) / 1000.0D), (diffHistogram.getValueAtPercentile(99.9) / 1000.0D), (diffHistogram.getValueAtPercentile(99.99) / 1000.0D), (diffHistogram.getValueAtPercentile(99.999) / 1000.0D));
            } else {
                System.out.printf("%12s, %10d, %10d, %8.2fms, %8.2fms, %8.2fms, %8.2fms, %8.2fms\n", sdf.format(now), totalCount, 0, 0D, 0D, 0D, 0D, 0D);
            }
            m_histogramData = AbstractHistogram.fromCompressedBytes(table.getVarbinary(4), CompressionStrategySnappy.INSTANCE);
            ;
        }
    }, 0, duration, TimeUnit.SECONDS);
}

Example 14 with ClientConfig

use of org.voltdb.client.ClientConfig in project voltdb by VoltDB.

the class KafkaLoader method processKafkaMessages.

public void processKafkaMessages() throws Exception {
    // Split server list
    final String[] serverlist = m_config.servers.split(",");
    // If we need to prompt the user for a VoltDB password, do so.
    m_config.password = CLIConfig.readPasswordIfNeeded(m_config.user, m_config.password, "Enter password: ");
    // Create connection
    final ClientConfig c_config = new ClientConfig(m_config.user, m_config.password, null);
    if (m_config.ssl != null && !m_config.ssl.trim().isEmpty()) {
        c_config.setTrustStoreConfigFromPropertyFile(m_config.ssl);
        c_config.enableSSL();
    }
    // Set procedure all to infinite
    c_config.setProcedureCallTimeout(0);
    m_client = getClient(c_config, serverlist, m_config.port);
    if (m_config.useSuppliedProcedure) {
        m_loader = new CSVTupleDataLoader((ClientImpl) m_client, m_config.procedure, new KafkaBulkLoaderCallback());
    } else {
        m_loader = new CSVBulkDataLoader((ClientImpl) m_client, m_config.table, m_config.batch, m_config.update, new KafkaBulkLoaderCallback());
    }
    m_loader.setFlushInterval(m_config.flush, m_config.flush);
    m_consumer = new KafkaConsumerConnector(m_config);
    try {
        m_executorService = getConsumerExecutor(m_consumer, m_loader);
        if (m_config.useSuppliedProcedure) {
            m_log.info("Kafka Consumer from topic: " + m_config.topic + " Started using procedure: " + m_config.procedure);
        } else {
            m_log.info("Kafka Consumer from topic: " + m_config.topic + " Started for table: " + m_config.table);
        }
        m_executorService.awaitTermination(365, TimeUnit.DAYS);
    } catch (Throwable terminate) {
        m_log.error("Error in Kafka Consumer", terminate);
        System.exit(-1);
    }
    close();
}

Example 15 with ClientConfig

use of org.voltdb.client.ClientConfig in project voltdb by VoltDB.

the class JDBCLoader method main.

/**
     * jdbcloader main. (main is directly used by tests as well be sure to reset statics that you need to start over)
     *
     * @param args
     * @throws IOException
     * @throws InterruptedException
     *
     */
public static void main(String[] args) throws IOException, InterruptedException {
    start = System.currentTimeMillis();
    long insertTimeStart = start;
    long insertTimeEnd;
    final JDBCLoaderConfig cfg = new JDBCLoaderConfig();
    cfg.parse(JDBCLoader.class.getName(), args);
    m_config = cfg;
    configuration();
    // Split server list
    final String[] serverlist = m_config.servers.split(",");
    // If we need to prompt the user for a VoltDB password, do so.
    m_config.password = CLIConfig.readPasswordIfNeeded(m_config.user, m_config.password, "Enter VoltDB password: ");
    // Create connection
    final ClientConfig c_config = new ClientConfig(m_config.user, m_config.password, null);
    if (m_config.ssl != null && !m_config.ssl.trim().isEmpty()) {
        c_config.setTrustStoreConfigFromPropertyFile(m_config.ssl);
        c_config.enableSSL();
    }
    // Set procedure all to infinite
    c_config.setProcedureCallTimeout(0);
    Client csvClient = null;
    try {
        csvClient = JDBCLoader.getClient(c_config, serverlist, m_config.port);
    } catch (Exception e) {
        m_log.error("Error connecting to the servers: " + m_config.servers, e);
        System.exit(-1);
    }
    assert (csvClient != null);
    try {
        long readerTime;
        long insertCount;
        long ackCount;
        final JDBCLoader errHandler = new JDBCLoader();
        final CSVDataLoader dataLoader;
        errHandler.launchErrorFlushProcessor();
        if (m_config.useSuppliedProcedure) {
            dataLoader = new CSVTupleDataLoader((ClientImpl) csvClient, m_config.procedure, errHandler);
        } else {
            dataLoader = new CSVBulkDataLoader((ClientImpl) csvClient, m_config.table, m_config.batch, m_config.update, errHandler);
        }
        // If we need to prompt the user for a JDBC datasource password, do so.
        m_config.jdbcpassword = CLIConfig.readPasswordIfNeeded(m_config.jdbcuser, m_config.jdbcpassword, "Enter JDBC source database password: ");
        //Created Source reader
        JDBCStatementReader.initializeReader(cfg, csvClient);
        JDBCStatementReader jdbcReader = new JDBCStatementReader(dataLoader, errHandler);
        Thread readerThread = new Thread(jdbcReader);
        readerThread.setName("JDBCSourceReader");
        readerThread.setDaemon(true);
        //Wait for reader to finish.
        readerThread.start();
        readerThread.join();
        insertTimeEnd = System.currentTimeMillis();
        csvClient.close();
        errHandler.waitForErrorFlushComplete();
        readerTime = (jdbcReader.m_parsingTime) / 1000000;
        insertCount = dataLoader.getProcessedRows();
        ackCount = insertCount - dataLoader.getFailedRows();
        if (errHandler.hasReachedErrorLimit()) {
            m_log.warn("The number of failed rows exceeds the configured maximum failed rows: " + m_config.maxerrors);
        }
        if (m_log.isDebugEnabled()) {
            m_log.debug("Parsing CSV file took " + readerTime + " milliseconds.");
            m_log.debug("Inserting Data took " + ((insertTimeEnd - insertTimeStart) - readerTime) + " milliseconds.");
        }
        m_log.info("Read " + insertCount + " rows from file and successfully inserted " + ackCount + " rows (final)");
        errHandler.produceFiles(ackCount, insertCount);
        close_cleanup();
        //In test junit mode we let it continue for reuse
        if (!JDBCLoader.testMode) {
            System.exit(errHandler.m_errorInfo.isEmpty() ? 0 : -1);
        }
    } catch (Exception ex) {
        m_log.error("Exception Happened while loading CSV data", ex);
        System.exit(1);
    }
}