use of com.amazonaws.services.s3.model.Region in project jackrabbit-oak by apache.
the class S3Backend method init.
public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
startTime = new Date();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("init");
this.store = store;
s3ReqDecorator = new S3RequestDecorator(prop);
s3service = Utils.openService(prop);
if (bucket == null || "".equals(bucket.trim())) {
bucket = prop.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (Strings.isNullOrEmpty(bucket)) {
bucket = prop.getProperty(S3Constants.S3_CONTAINER);
}
}
String region = prop.getProperty(S3Constants.S3_REGION);
Region s3Region = null;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
if (ec2Region != null) {
s3Region = Region.fromValue(ec2Region.getName());
} else {
throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
}
} else {
if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
s3Region = Region.US_Standard;
} else if (Region.EU_Ireland.toString().equals(region)) {
s3Region = Region.EU_Ireland;
} else {
s3Region = Region.fromValue(region);
}
}
if (!s3service.doesBucketExist(bucket)) {
s3service.createBucket(bucket, s3Region);
LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
int writeThreads = 10;
String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
if (writeThreadsStr != null) {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
int asyncWritePoolSize = 10;
String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
if (maxConnsStr != null) {
asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
}
asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize, new NamedThreadFactory("s3-write-worker"));
String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
}
LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
LOG.debug(" error ", e);
Map<String, String> filteredMap = Maps.newHashMap();
if (prop != null) {
filteredMap = Maps.filterKeys(Maps.fromProperties(prop), new Predicate<String>() {
@Override
public boolean apply(String input) {
return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
}
});
}
throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of com.amazonaws.services.s3.model.Region in project jackrabbit-oak by apache.
the class S3Backend method init.
public void init() throws DataStoreException {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
startTime = new Date();
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
LOG.debug("init");
s3ReqDecorator = new S3RequestDecorator(properties);
s3service = Utils.openService(properties);
if (bucket == null || "".equals(bucket.trim())) {
bucket = properties.getProperty(S3Constants.S3_BUCKET);
// Alternately check if the 'container' property is set
if (Strings.isNullOrEmpty(bucket)) {
bucket = properties.getProperty(S3Constants.S3_CONTAINER);
}
}
secret = properties.getProperty("secret");
String region = properties.getProperty(S3Constants.S3_REGION);
Region s3Region;
if (StringUtils.isNullOrEmpty(region)) {
com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
if (ec2Region != null) {
s3Region = Region.fromValue(ec2Region.getName());
} else {
throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
}
} else {
if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
s3Region = Region.US_Standard;
} else if (Region.EU_Ireland.toString().equals(region)) {
s3Region = Region.EU_Ireland;
} else {
s3Region = Region.fromValue(region);
}
}
if (!s3service.doesBucketExist(bucket)) {
s3service.createBucket(bucket, s3Region);
LOG.info("Created bucket [{}] in [{}] ", bucket, region);
} else {
LOG.info("Using bucket [{}] in [{}] ", bucket, region);
}
int writeThreads = 10;
String writeThreadsStr = properties.getProperty(S3Constants.S3_WRITE_THREADS);
if (writeThreadsStr != null) {
writeThreads = Integer.parseInt(writeThreadsStr);
}
LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
tmx = new TransferManager(s3service, Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
String renameKeyProp = properties.getProperty(S3Constants.S3_RENAME_KEYS);
boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
LOG.info("Rename keys [{}]", renameKeyBool);
if (renameKeyBool) {
renameKeys();
}
LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
} catch (Exception e) {
LOG.debug(" error ", e);
Map<String, String> filteredMap = Maps.newHashMap();
if (properties != null) {
filteredMap = Maps.filterKeys(Maps.fromProperties(properties), new Predicate<String>() {
@Override
public boolean apply(String input) {
return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
}
});
}
throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of com.amazonaws.services.s3.model.Region in project XRTB by benmfaul.
the class Configuration method initialize.
/**
* Read the Java Bean Shell file that initializes this constructor.
*
* @param path.
* String - The file name containing the Java Bean Shell code.
* @throws Exception
* on file errors.
*/
public void initialize(String path, String shard, int port, int sslPort) throws Exception {
this.fileName = path;
/******************************
* System Name
*****************************/
this.shard = shard;
this.port = port;
this.sslPort = sslPort;
java.net.InetAddress localMachine = null;
String useName = null;
try {
localMachine = java.net.InetAddress.getLocalHost();
ipAddress = localMachine.getHostAddress();
useName = localMachine.getHostName();
} catch (Exception error) {
useName = getIpAddress();
}
if (shard == null || shard.length() == 0)
instanceName = useName + ":" + port;
else
instanceName = shard + ":" + useName + ":" + port;
/**
* Set up tem p files
*/
// create the temp
Files.createDirectories(Paths.get("www/temp"));
// directory in www so
// preview campaign will
// work
/*********************************************
* USE ZOOKEEPER, AEROSPIKE OR FILE CONFIG
*********************************************/
String str = null;
if (path.startsWith("zookeeper")) {
String[] parts = path.split(":");
System.out.println(parts);
zk = new ZkConnect(parts[1]);
zk.join(parts[2], "bidders", instanceName);
str = zk.readConfig(parts[2] + "/bidders");
} else if (path.startsWith("aerospike")) {
String[] parts = path.split(":");
System.out.println(parts);
String aerospike = parts[1];
String configKey = parts[2];
AerospikeHandler spike = AerospikeHandler.getInstance(aerospike, 3000, 300);
redisson = new RedissonClient(spike);
Database.getInstance(redisson);
str = redisson.get(configKey);
if (str == null) {
throw new Exception("Aerospike configuration at " + path + " not available.");
}
System.out.println(str);
} else {
byte[] encoded = Files.readAllBytes(Paths.get(path));
str = Charset.defaultCharset().decode(ByteBuffer.wrap(encoded)).toString();
}
Map<?, ?> m = DbTools.mapper.readValue(str, Map.class);
/*******************************************************************************/
seats = new HashMap<String, String>();
if (m.get("lists") != null) {
filesList = (List) m.get("lists");
initializeLookingGlass(filesList);
}
if (m.get("s3") != null) {
Map<String, String> ms3 = (Map) m.get("s3");
String accessKey = ms3.get("access_key_id");
String secretAccessKey = ms3.get("secret_access_key");
String region = ms3.get("region");
s3_bucket = ms3.get("bucket");
s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(s3_bucket));
try {
processDirectory(s3, listing, s3_bucket);
} catch (Exception error) {
System.err.println("ERROR IN AWS LISTING: " + error.toString());
}
}
if (m.get("ssl") != null) {
Map x = (Map) m.get("ssl");
ssl = new SSL();
ssl.setKeyManagerPassword = (String) x.get("setKeyManagerPassword");
ssl.setKeyStorePassword = (String) x.get("setKeyStorePassword");
ssl.setKeyStorePath = (String) x.get("setKeyStorePath");
}
/**
* Create the seats id map, and create the bin and win handler classes
* for each exchange
*/
seatsList = (List<Map>) m.get("seats");
for (int i = 0; i < seatsList.size(); i++) {
Map x = seatsList.get(i);
String seatId = (String) x.get("id");
String className = (String) x.get("bid");
int k = className.indexOf("=");
String[] parts = new String[2];
String uri = className.substring(0, k);
className = className.substring(k + 1);
String[] options = null;
/**
* set up any options on the class string
*/
if (className.contains("&")) {
parts = className.split("&");
className = parts[0].trim();
options = parts[1].split(",");
for (int ind = 0; ind < options.length; ind++) {
options[ind] = options[ind].trim();
}
}
String[] tags = uri.split("/");
String exchange = tags[tags.length - 1];
String name = (String) x.get("name");
if (name == null)
name = exchange;
String id = (String) x.get("id");
seats.put(name, id);
try {
Class<?> c = Class.forName(className);
BidRequest br = (BidRequest) c.newInstance();
if (br == null) {
throw new Exception("Could not make new instance of: " + className);
}
Map extension = (Map) x.get("extension");
if (x != null)
br.handleConfigExtensions(extension);
/**
* Handle generic-ized exchanges
*/
if (className.contains("Generic")) {
br.setExchange(exchange);
br.usesEncodedAdm = true;
}
RTBServer.exchanges.put(uri, br);
if (parts[0] != null) {
for (int ind = 1; ind < parts.length; ind++) {
String option = parts[ind];
String[] tuples = option.split("=");
switch(tuples[0]) {
case "usesEncodedAdm":
br.usesEncodedAdm = true;
break;
case "!usesEncodedAdm":
br.usesEncodedAdm = false;
break;
case "rlog":
Double rlog = Double.parseDouble(tuples[1]);
ExchangeLogLevel.getInstance().setExchangeLogLevel(name, rlog.intValue());
break;
case "useStrings":
break;
case "!useStrings":
break;
default:
System.err.println("Unknown request: " + tuples[0] + " in definition of " + className);
}
}
}
/**
* Appnexus requires additional support for ready, pixel and
* click
*/
if (className.contains("Appnexus")) {
RTBServer.exchanges.put(uri + "/ready", new Appnexus(Appnexus.READY));
RTBServer.exchanges.put(uri + "/pixel", new Appnexus(Appnexus.PIXEL));
RTBServer.exchanges.put(uri + "/click", new Appnexus(Appnexus.CLICK));
RTBServer.exchanges.put(uri + "/delivered", new Appnexus(Appnexus.DELIVERED));
Appnexus.seatId = seatId;
}
} catch (Exception error) {
System.err.println("Error configuring exchange: " + name + ", error = ");
throw error;
}
}
/**
* Create forensiq
*/
Map fraud = (Map) m.get("fraud");
if (fraud != null) {
if (m.get("forensiq") != null) {
System.out.println("*** Fraud detection is set to Forensiq");
Map f = (Map) m.get("forensiq");
String ck = (String) f.get("ck");
Integer x = (Integer) f.get("threshhold");
if (!(x == 0 || ck == null || ck.equals("none"))) {
ForensiqClient fx = ForensiqClient.build(ck);
if (fraud.get("endpoint") != null) {
fx.endpoint = (String) fraud.get("endpoint");
}
if (fraud.get("bidOnError") != null) {
fx.bidOnError = (Boolean) fraud.get("bidOnError");
}
if (f.get("connections") != null)
ForensiqClient.getInstance().connections = (int) (Integer) fraud.get("connections");
forensiq = fx;
}
} else {
System.out.println("*** Fraud detection is set to MMDB");
String db = (String) fraud.get("db");
MMDBClient fy = MMDBClient.build(db);
if (fraud.get("bidOnError") != null) {
fy.bidOnError = (Boolean) fraud.get("bidOnError");
}
if (fraud.get("watchlist") != null) {
fy.setWatchlist((List<String>) fraud.get("watchlist"));
}
forensiq = fy;
}
} else {
System.out.println("*** NO Fraud detection");
}
/**
* Deal with the app object
*/
m = (Map) m.get("app");
password = (String) m.get("password");
if (m.get("threads") != null) {
RTBServer.threads = (Integer) m.get("threads");
}
if (m.get("multibid") != null) {
multibid = (Boolean) m.get("multibid");
}
if (m.get("adminPort") != null) {
adminPort = (Integer) m.get("adminPort");
}
if (m.get("adminSSL") != null) {
adminSSL = (Boolean) m.get("adminSSL");
}
String strategy = (String) m.get("strategy");
if (strategy != null && strategy.equals("heuristic"))
RTBServer.strategy = STRATEGY_HEURISTIC;
else
RTBServer.strategy = STRATEGY_MAX_CONNECTIONS;
verbosity = (Map) m.get("verbosity");
if (verbosity != null) {
logLevel = (Integer) verbosity.get("level");
printNoBidReason = (Boolean) verbosity.get("nobid-reason");
}
template = (Map) m.get("template");
if (template == null) {
throw new Exception("No template defined");
}
encodeTemplates();
encodeTemplateStubs();
geotags = (Map) m.get("geotags");
if (geotags != null) {
String states = (String) geotags.get("states");
String codes = (String) geotags.get("zipcodes");
geoTagger.initTags(states, codes);
}
Boolean bValue = false;
bValue = (Boolean) m.get("stopped");
if (bValue != null && bValue == true) {
RTBServer.stopped = true;
pauseOnStart = true;
}
Map redis = (Map) m.get("redis");
if (redis != null) {
Integer rsize = (Integer) redis.get("pool");
if (rsize == null)
rsize = 64;
String host = (String) redis.get("host");
Integer rport = (Integer) redis.get("port");
if (rport == null)
rport = 6379;
// JedisPoolConfig poolConfig = new JedisPoolConfig();;
// configJedis.setMaxTotal(rsize);
// configJedis.setMaxWaitMillis(10);
// poolConfig.setMaxIdle(4000);
// Tests whether connections are dead during idle periods
// poolConfig.setTestWhileIdle(true);
// poolConfig.setMaxTotal(4000);
// poolConfig.setMaxWaitMillis(30);
// jedisPool = new JedisPool(poolConfig,host,rport);
MyJedisPool.host = host;
MyJedisPool.port = rport;
jedisPool = new MyJedisPool(1000, 1000, 5);
System.out.println("*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + rport + ", size = " + rsize);
}
Map zeromq = (Map) m.get("zeromq");
String value = null;
Double dValue = 0.0;
bValue = false;
Map r = (Map) m.get("aerospike");
if (r != null) {
if ((value = (String) r.get("host")) != null)
cacheHost = value;
if (r.get("port") != null)
cachePort = (Integer) r.get("port");
if (r.get("maxconns") != null)
maxconns = (Integer) r.get("maxconns");
AerospikeHandler.getInstance(cacheHost, cachePort, maxconns);
redisson = new RedissonClient(AerospikeHandler.getInstance());
Database.getInstance(redisson);
System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + maxconns + ", handlers: " + AerospikeHandler.getInstance().getCount() + " ***");
String key = (String) m.get("deadmanswitch");
if (key != null) {
deadmanSwitch = new DeadmanSwitch(redisson, key);
}
} else {
redisson = new RedissonClient();
Database db = Database.getInstance(redisson);
readDatabaseIntoCache("database.json");
readBlackListIntoCache("blacklist.json");
}
/**
* Zeromq
*/
if ((value = (String) zeromq.get("bidchannel")) != null)
BIDS_CHANNEL = value;
if ((value = (String) zeromq.get("nobidchannel")) != null)
NOBIDS_CHANNEL = value;
if ((value = (String) zeromq.get("winchannel")) != null)
WINS_CHANNEL = value;
if ((value = (String) zeromq.get("requests")) != null)
REQUEST_CHANNEL = value;
if ((value = (String) zeromq.get("unilogger")) != null)
UNILOGGER_CHANNEL = value;
if ((value = (String) zeromq.get("logger")) != null)
LOG_CHANNEL = value;
if ((value = (String) zeromq.get("clicks")) != null)
CLICKS_CHANNEL = value;
if ((value = (String) zeromq.get("fraud")) != null)
FORENSIQ_CHANNEL = value;
if ((value = (String) zeromq.get("responses")) != null)
RESPONSES = value;
if ((value = (String) zeromq.get("status")) != null)
PERF_CHANNEL = value;
if ((value = (String) zeromq.get("reasons")) != null)
REASONS_CHANNEL = value;
Map xx = (Map) zeromq.get("subscribers");
List<String> list = (List) xx.get("hosts");
commandsPort = (String) xx.get("commands");
for (String host : list) {
String address = "tcp://" + host + ":" + commandsPort + "&commands";
commandAddresses.add(address);
}
if (zeromq.get("requeststrategy") != null) {
Object obj = zeromq.get("requeststrategy");
if (obj instanceof String) {
strategy = (String) zeromq.get("requeststrategy");
if (strategy.equalsIgnoreCase("all") || strategy.equalsIgnoreCase("requests"))
requstLogStrategy = REQUEST_STRATEGY_ALL;
if (strategy.equalsIgnoreCase("bids"))
requstLogStrategy = REQUEST_STRATEGY_BIDS;
if (strategy.equalsIgnoreCase("WINS"))
requstLogStrategy = REQUEST_STRATEGY_WINS;
} else {
if (obj instanceof Integer) {
int level = (Integer) obj;
ExchangeLogLevel.getInstance().setStdLevel(level);
} else if (obj instanceof Double) {
Double perc = (Double) obj;
ExchangeLogLevel.getInstance().setStdLevel(perc.intValue());
}
}
}
/********************************************************************/
campaignsList.clear();
pixelTrackingUrl = (String) m.get("pixel-tracking-url");
winUrl = (String) m.get("winurl");
redirectUrl = (String) m.get("redirect-url");
if (m.get("ttl") != null) {
ttl = (Integer) m.get("ttl");
}
initialLoadlist = (List<Map>) m.get("campaigns");
for (Map<String, String> camp : initialLoadlist) {
if (camp.get("id") != null) {
addCampaign(camp.get("name"), camp.get("id"));
} else {
Controller.getInstance().sendLog(1, "Configuration", "*** ERRORS DETECTED IN INITIAL LOAD OF CAMPAIGNS *** ");
}
}
if (cacheHost == null)
Controller.getInstance().sendLog(1, "Configuration", "*** NO AEROSPIKE CONFIGURED, USING CACH2K INSTEAD *** ");
if (winUrl.contains("localhost")) {
Controller.getInstance().sendLog(1, "Configuration", "*** WIN URL IS SET TO LOCALHOST, NO REMOTE ACCESS WILL WORK FOR WINS ***");
}
}
use of com.amazonaws.services.s3.model.Region in project h2o-3 by h2oai.
the class PersistS3 method configureClient.
static AmazonS3Client configureClient(AmazonS3Client s3Client) {
if (System.getProperty(S3_REGION) != null) {
String region = System.getProperty(S3_REGION);
Log.debug("S3 region specified: ", region);
s3Client.setRegion(RegionUtils.getRegion(region));
}
// Region overrides end-point settings
if (System.getProperty(S3_END_POINT) != null) {
String endPoint = System.getProperty(S3_END_POINT);
Log.debug("S3 endpoint specified: ", endPoint);
s3Client.setEndpoint(endPoint);
}
if (System.getProperty(S3_ENABLE_PATH_STYLE) != null && Boolean.valueOf(System.getProperty(S3_ENABLE_PATH_STYLE))) {
Log.debug("S3 path style access enabled");
S3ClientOptions sco = new S3ClientOptions();
sco.setPathStyleAccess(true);
s3Client.setS3ClientOptions(sco);
}
return s3Client;
}
use of com.amazonaws.services.s3.model.Region in project camel by apache.
the class S3Endpoint method doStart.
@Override
public void doStart() throws Exception {
super.doStart();
s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : createS3Client();
if (ObjectHelper.isNotEmpty(configuration.getAmazonS3Endpoint())) {
s3Client.setEndpoint(configuration.getAmazonS3Endpoint());
}
String fileName = getConfiguration().getFileName();
if (fileName != null) {
LOG.trace("File name [{}] requested, so skipping bucket check...", fileName);
return;
}
String bucketName = getConfiguration().getBucketName();
LOG.trace("Querying whether bucket [{}] already exists...", bucketName);
String prefix = getConfiguration().getPrefix();
try {
s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0));
LOG.trace("Bucket [{}] already exists", bucketName);
return;
} catch (AmazonServiceException ase) {
/* 404 means the bucket doesn't exist */
if (ase.getStatusCode() != 404) {
throw ase;
}
}
LOG.trace("Bucket [{}] doesn't exist yet", bucketName);
// creates the new bucket because it doesn't exist yet
CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName());
if (getConfiguration().getRegion() != null) {
createBucketRequest.setRegion(getConfiguration().getRegion());
}
LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest);
s3Client.createBucket(createBucketRequest);
LOG.trace("Bucket created");
if (configuration.getPolicy() != null) {
LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy());
s3Client.setBucketPolicy(bucketName, configuration.getPolicy());
LOG.trace("Bucket policy updated");
}
}
Aggregations