Examples with DBException site.ycsb.DBException used on opensource projects

Example 6 with DBException

use of site.ycsb.DBException in project YCSB by brianfrankcooper.

the class S3Client method init.

/**
 * Initialize any state for the storage.
 * Called once per S3 instance; If the client is not null it is re-used.
 */
@Override
public void init() throws DBException {
    final int count = INIT_COUNT.incrementAndGet();
    synchronized (S3Client.class) {
        Properties propsCL = getProperties();
        int recordcount = Integer.parseInt(propsCL.getProperty("recordcount"));
        int operationcount = Integer.parseInt(propsCL.getProperty("operationcount"));
        int numberOfOperations = 0;
        if (recordcount > 0) {
            if (recordcount > operationcount) {
                numberOfOperations = recordcount;
            } else {
                numberOfOperations = operationcount;
            }
        } else {
            numberOfOperations = operationcount;
        }
        if (count <= numberOfOperations) {
            String accessKeyId = null;
            String secretKey = null;
            String endPoint = null;
            String region = null;
            String maxErrorRetry = null;
            String maxConnections = null;
            String protocol = null;
            BasicAWSCredentials s3Credentials;
            ClientConfiguration clientConfig;
            if (s3Client != null) {
                System.out.println("Reusing the same client");
                return;
            }
            try {
                InputStream propFile = S3Client.class.getClassLoader().getResourceAsStream("s3.properties");
                Properties props = new Properties(System.getProperties());
                props.load(propFile);
                accessKeyId = props.getProperty("s3.accessKeyId");
                if (accessKeyId == null) {
                    accessKeyId = propsCL.getProperty("s3.accessKeyId");
                }
                System.out.println(accessKeyId);
                secretKey = props.getProperty("s3.secretKey");
                if (secretKey == null) {
                    secretKey = propsCL.getProperty("s3.secretKey");
                }
                System.out.println(secretKey);
                endPoint = props.getProperty("s3.endPoint");
                if (endPoint == null) {
                    endPoint = propsCL.getProperty("s3.endPoint", "s3.amazonaws.com");
                }
                System.out.println(endPoint);
                region = props.getProperty("s3.region");
                if (region == null) {
                    region = propsCL.getProperty("s3.region", "us-east-1");
                }
                System.out.println(region);
                maxErrorRetry = props.getProperty("s3.maxErrorRetry");
                if (maxErrorRetry == null) {
                    maxErrorRetry = propsCL.getProperty("s3.maxErrorRetry", "15");
                }
                maxConnections = props.getProperty("s3.maxConnections");
                if (maxConnections == null) {
                    maxConnections = propsCL.getProperty("s3.maxConnections");
                }
                protocol = props.getProperty("s3.protocol");
                if (protocol == null) {
                    protocol = propsCL.getProperty("s3.protocol", "HTTPS");
                }
                sse = props.getProperty("s3.sse");
                if (sse == null) {
                    sse = propsCL.getProperty("s3.sse", "false");
                }
                String ssec = props.getProperty("s3.ssec");
                if (ssec == null) {
                    ssec = propsCL.getProperty("s3.ssec", null);
                } else {
                    ssecKey = new SSECustomerKey(ssec);
                }
            } catch (Exception e) {
                System.err.println("The file properties doesn't exist " + e.toString());
                e.printStackTrace();
            }
            try {
                System.out.println("Inizializing the S3 connection");
                s3Credentials = new BasicAWSCredentials(accessKeyId, secretKey);
                clientConfig = new ClientConfiguration();
                clientConfig.setMaxErrorRetry(Integer.parseInt(maxErrorRetry));
                if (protocol.equals("HTTP")) {
                    clientConfig.setProtocol(Protocol.HTTP);
                } else {
                    clientConfig.setProtocol(Protocol.HTTPS);
                }
                if (maxConnections != null) {
                    clientConfig.setMaxConnections(Integer.parseInt(maxConnections));
                }
                s3Client = new AmazonS3Client(s3Credentials, clientConfig);
                s3Client.setRegion(Region.getRegion(Regions.fromName(region)));
                s3Client.setEndpoint(endPoint);
                System.out.println("Connection successfully initialized");
            } catch (Exception e) {
                System.err.println("Could not connect to S3 storage: " + e.toString());
                e.printStackTrace();
                throw new DBException(e);
            }
        } else {
            System.err.println("The number of threads must be less or equal than the operations");
            throw new DBException(new Error("The number of threads must be less or equal than the operations"));
        }
    }
}

Example 7 with DBException

use of site.ycsb.DBException in project YCSB by brianfrankcooper.

the class ScyllaCQLClient method init.

/**
 * Initialize any state for this DB. Called once per DB instance; there is one
 * DB instance per client thread.
 */
@Override
public void init() throws DBException {
    // Keep track of number of calls to init (for later cleanup)
    INIT_COUNT.incrementAndGet();
    // cluster/session instance for all the threads.
    synchronized (INIT_COUNT) {
        // Check if the cluster has already been initialized
        if (cluster != null) {
            return;
        }
        try {
            debug = Boolean.parseBoolean(getProperties().getProperty("debug", "false"));
            trace = Boolean.parseBoolean(getProperties().getProperty(TRACING_PROPERTY, TRACING_PROPERTY_DEFAULT));
            String host = getProperties().getProperty(HOSTS_PROPERTY);
            if (host == null) {
                throw new DBException(String.format("Required property \"%s\" missing for scyllaCQLClient", HOSTS_PROPERTY));
            }
            String[] hosts = host.split(",");
            String port = getProperties().getProperty(PORT_PROPERTY, PORT_PROPERTY_DEFAULT);
            String username = getProperties().getProperty(USERNAME_PROPERTY);
            String password = getProperties().getProperty(PASSWORD_PROPERTY);
            String keyspace = getProperties().getProperty(KEYSPACE_PROPERTY, KEYSPACE_PROPERTY_DEFAULT);
            readConsistencyLevel = ConsistencyLevel.valueOf(getProperties().getProperty(READ_CONSISTENCY_LEVEL_PROPERTY, READ_CONSISTENCY_LEVEL_PROPERTY_DEFAULT));
            writeConsistencyLevel = ConsistencyLevel.valueOf(getProperties().getProperty(WRITE_CONSISTENCY_LEVEL_PROPERTY, WRITE_CONSISTENCY_LEVEL_PROPERTY_DEFAULT));
            boolean useSSL = Boolean.parseBoolean(getProperties().getProperty(USE_SSL_CONNECTION, DEFAULT_USE_SSL_CONNECTION));
            Cluster.Builder builder;
            if ((username != null) && !username.isEmpty()) {
                builder = Cluster.builder().withCredentials(username, password).addContactPoints(hosts).withPort(Integer.parseInt(port));
                if (useSSL) {
                    builder = builder.withSSL();
                }
            } else {
                builder = Cluster.builder().withPort(Integer.parseInt(port)).addContactPoints(hosts);
            }
            final String localDC = getProperties().getProperty(TOKEN_AWARE_LOCAL_DC);
            if (localDC != null && !localDC.isEmpty()) {
                final LoadBalancingPolicy local = DCAwareRoundRobinPolicy.builder().withLocalDc(localDC).build();
                final TokenAwarePolicy tokenAware = new TokenAwarePolicy(local);
                builder = builder.withLoadBalancingPolicy(tokenAware);
                LOGGER.info("Using local datacenter with token awareness: {}\n", localDC);
                // If was not overridden explicitly, set LOCAL_QUORUM
                if (getProperties().getProperty(READ_CONSISTENCY_LEVEL_PROPERTY) == null) {
                    readConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM;
                }
                if (getProperties().getProperty(WRITE_CONSISTENCY_LEVEL_PROPERTY) == null) {
                    writeConsistencyLevel = ConsistencyLevel.LOCAL_QUORUM;
                }
            }
            cluster = builder.build();
            String maxConnections = getProperties().getProperty(MAX_CONNECTIONS_PROPERTY);
            if (maxConnections != null) {
                cluster.getConfiguration().getPoolingOptions().setMaxConnectionsPerHost(HostDistance.LOCAL, Integer.parseInt(maxConnections));
            }
            String coreConnections = getProperties().getProperty(CORE_CONNECTIONS_PROPERTY);
            if (coreConnections != null) {
                cluster.getConfiguration().getPoolingOptions().setCoreConnectionsPerHost(HostDistance.LOCAL, Integer.parseInt(coreConnections));
            }
            String connectTimeoutMillis = getProperties().getProperty(CONNECT_TIMEOUT_MILLIS_PROPERTY);
            if (connectTimeoutMillis != null) {
                cluster.getConfiguration().getSocketOptions().setConnectTimeoutMillis(Integer.parseInt(connectTimeoutMillis));
            }
            String readTimeoutMillis = getProperties().getProperty(READ_TIMEOUT_MILLIS_PROPERTY);
            if (readTimeoutMillis != null) {
                cluster.getConfiguration().getSocketOptions().setReadTimeoutMillis(Integer.parseInt(readTimeoutMillis));
            }
            Metadata metadata = cluster.getMetadata();
            LOGGER.info("Connected to cluster: {}\n", metadata.getClusterName());
            for (Host discoveredHost : metadata.getAllHosts()) {
                LOGGER.info("Datacenter: {}; Host: {}; Rack: {}\n", discoveredHost.getDatacenter(), discoveredHost.getEndPoint().resolve().getAddress(), discoveredHost.getRack());
            }
            session = cluster.connect(keyspace);
            if (Boolean.parseBoolean(getProperties().getProperty(SCYLLA_LWT, Boolean.toString(lwt)))) {
                LOGGER.info("Using LWT\n");
                lwt = true;
                readConsistencyLevel = ConsistencyLevel.SERIAL;
                writeConsistencyLevel = ConsistencyLevel.ANY;
            } else {
                LOGGER.info("Not using LWT\n");
            }
            LOGGER.info("Read consistency: {}, Write consistency: {}\n", readConsistencyLevel.name(), writeConsistencyLevel.name());
        } catch (Exception e) {
            throw new DBException(e);
        }
    }
// synchronized
}

Example 8 with DBException

use of site.ycsb.DBException in project YCSB by brianfrankcooper.

the class VoltClient4 method init.

@Override
public void init() throws DBException {
    Properties props = getProperties();
    String servers = props.getProperty("voltdb.servers", "localhost");
    String user = props.getProperty("voltdb.user", "");
    String password = props.getProperty("voltdb.password", "");
    String strLimit = props.getProperty("voltdb.ratelimit");
    String useScanAllParam = props.getProperty("voltdb.scanall", "no");
    if (useScanAllParam.equalsIgnoreCase("YES")) {
        useScanAll = true;
    }
    int ratelimit = strLimit != null ? Integer.parseInt(strLimit) : Integer.MAX_VALUE;
    try {
        mclient = ConnectionHelper.createConnection(servers, user, password, ratelimit);
        ysb = StaticHolder.INSTANCE;
        ysb.loadClassesAndDDLIfNeeded(mclient);
    } catch (Exception e) {
        logger.error("Error while creating connection: ", e);
        throw new DBException(e.getMessage());
    }
    mworkingData = new byte[1024 * 1024];
    mwriteBuf = ByteBuffer.wrap(mworkingData);
}

Example 9 with DBException

use of site.ycsb.DBException in project YCSB by brianfrankcooper.

the class HBaseClient2 method cleanup.

/**
 * Cleanup any state for this DB. Called once per DB instance; there is one DB
 * instance per client thread.
 */
@Override
public void cleanup() throws DBException {
    // Get the measurements instance as this is the only client that should
    // count clean up time like an update if client-side buffering is
    // enabled.
    Measurements measurements = Measurements.getMeasurements();
    try {
        long st = System.nanoTime();
        if (bufferedMutator != null) {
            bufferedMutator.close();
        }
        if (currentTable != null) {
            currentTable.close();
        }
        long en = System.nanoTime();
        final String type = clientSideBuffering ? "UPDATE" : "CLEANUP";
        measurements.measure(type, (int) ((en - st) / 1000));
        int threadCount = THREAD_COUNT.decrementAndGet();
        if (threadCount <= 0) {
            // Means we are done so ok to shut down the Connection.
            synchronized (THREAD_COUNT) {
                if (connection != null) {
                    connection.close();
                    connection = null;
                }
            }
        }
    } catch (IOException e) {
        throw new DBException(e);
    }
}

Example 10 with DBException

use of site.ycsb.DBException in project YCSB by brianfrankcooper.

the class HBaseClient2 method init.

/**
 * Initialize any state for this DB. Called once per DB instance; there is one
 * DB instance per client thread.
 */
@Override
public void init() throws DBException {
    if ("true".equals(getProperties().getProperty("clientbuffering", "false"))) {
        this.clientSideBuffering = true;
    }
    if (getProperties().containsKey("writebuffersize")) {
        writeBufferSize = Long.parseLong(getProperties().getProperty("writebuffersize"));
    }
    if (getProperties().getProperty("durability") != null) {
        this.durability = Durability.valueOf(getProperties().getProperty("durability"));
    }
    if ("kerberos".equalsIgnoreCase(config.get("hbase.security.authentication"))) {
        config.set("hadoop.security.authentication", "Kerberos");
        UserGroupInformation.setConfiguration(config);
    }
    if ((getProperties().getProperty("principal") != null) && (getProperties().getProperty("keytab") != null)) {
        try {
            UserGroupInformation.loginUserFromKeytab(getProperties().getProperty("principal"), getProperties().getProperty("keytab"));
        } catch (IOException e) {
            System.err.println("Keytab file is not readable or not found");
            throw new DBException(e);
        }
    }
    String table = getProperties().getProperty(TABLENAME_PROPERTY, TABLENAME_PROPERTY_DEFAULT);
    try {
        THREAD_COUNT.getAndIncrement();
        synchronized (THREAD_COUNT) {
            if (connection == null) {
                // Initialize if not set up already.
                connection = ConnectionFactory.createConnection(config);
                // Terminate right now if table does not exist, since the client
                // will not propagate this error upstream once the workload
                // starts.
                final TableName tName = TableName.valueOf(table);
                try (Admin admin = connection.getAdmin()) {
                    if (!admin.tableExists(tName)) {
                        throw new DBException("Table " + tName + " does not exists");
                    }
                }
            }
        }
    } catch (java.io.IOException e) {
        throw new DBException(e);
    }
    if ((getProperties().getProperty("debug") != null) && (getProperties().getProperty("debug").compareTo("true") == 0)) {
        debug = true;
    }
    usePageFilter = isBooleanParamSet("hbase.usepagefilter", usePageFilter);
    if (isBooleanParamSet("hbase.usescanvaluefiltering", false)) {
        useScanValueFiltering = true;
        String operator = getProperties().getProperty("hbase.scanfilteroperator");
        operator = operator == null || operator.trim().isEmpty() ? DEFAULT_SCAN_FILTER_OPERATOR : operator;
        scanFilterOperator = CompareOperator.valueOf(operator.toUpperCase());
        String filterValue = getProperties().getProperty("hbase.scanfiltervalue");
        filterValue = filterValue == null || filterValue.trim().isEmpty() ? DEFAULT_SCAN_FILTER_VALUE : filterValue;
        scanFilterValue = new BinaryComparator(Bytes.fromHex(filterValue));
    }
    columnFamily = getProperties().getProperty("columnfamily");
    if (columnFamily == null) {
        System.err.println("Error, must specify a columnfamily for HBase table");
        throw new DBException("No columnfamily specified");
    }
    columnFamilyBytes = Bytes.toBytes(columnFamily);
}