use of com.amazonaws.services.s3.model.ListObjectsRequest in project jackrabbit-oak by apache.
the class S3Backend method deleteAllMetadataRecords.
@Override
public void deleteAllMetadataRecords(String prefix) {
ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
try {
Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucket).withPrefix(addMetaKeyPrefix(prefix));
ObjectListing metaList = s3service.listObjects(listObjectsRequest);
List<DeleteObjectsRequest.KeyVersion> deleteList = new ArrayList<DeleteObjectsRequest.KeyVersion>();
for (S3ObjectSummary s3ObjSumm : metaList.getObjectSummaries()) {
deleteList.add(new DeleteObjectsRequest.KeyVersion(s3ObjSumm.getKey()));
}
if (deleteList.size() > 0) {
DeleteObjectsRequest delObjsReq = new DeleteObjectsRequest(bucket);
delObjsReq.setKeys(deleteList);
s3service.deleteObjects(delObjsReq);
}
} finally {
if (contextClassLoader != null) {
Thread.currentThread().setContextClassLoader(contextClassLoader);
}
}
}
use of com.amazonaws.services.s3.model.ListObjectsRequest in project XRTB by benmfaul.
the class Configuration method initialize.
/**
* Read the Java Bean Shell file that initializes this constructor.
*
* @param path.
* String - The file name containing the Java Bean Shell code.
* @throws Exception
* on file errors.
*/
public void initialize(String path, String shard, int port, int sslPort) throws Exception {
this.fileName = path;
/******************************
* System Name
*****************************/
this.shard = shard;
this.port = port;
this.sslPort = sslPort;
java.net.InetAddress localMachine = null;
String useName = null;
try {
localMachine = java.net.InetAddress.getLocalHost();
ipAddress = localMachine.getHostAddress();
useName = localMachine.getHostName();
} catch (Exception error) {
useName = getIpAddress();
}
if (shard == null || shard.length() == 0)
instanceName = useName + ":" + port;
else
instanceName = shard + ":" + useName + ":" + port;
/**
* Set up tem p files
*/
// create the temp
Files.createDirectories(Paths.get("www/temp"));
// directory in www so
// preview campaign will
// work
/*********************************************
* USE ZOOKEEPER, AEROSPIKE OR FILE CONFIG
*********************************************/
String str = null;
if (path.startsWith("zookeeper")) {
String[] parts = path.split(":");
System.out.println(parts);
zk = new ZkConnect(parts[1]);
zk.join(parts[2], "bidders", instanceName);
str = zk.readConfig(parts[2] + "/bidders");
} else if (path.startsWith("aerospike")) {
String[] parts = path.split(":");
System.out.println(parts);
String aerospike = parts[1];
String configKey = parts[2];
AerospikeHandler spike = AerospikeHandler.getInstance(aerospike, 3000, 300);
redisson = new RedissonClient(spike);
Database.getInstance(redisson);
str = redisson.get(configKey);
if (str == null) {
throw new Exception("Aerospike configuration at " + path + " not available.");
}
System.out.println(str);
} else {
byte[] encoded = Files.readAllBytes(Paths.get(path));
str = Charset.defaultCharset().decode(ByteBuffer.wrap(encoded)).toString();
}
Map<?, ?> m = DbTools.mapper.readValue(str, Map.class);
/*******************************************************************************/
seats = new HashMap<String, String>();
if (m.get("lists") != null) {
filesList = (List) m.get("lists");
initializeLookingGlass(filesList);
}
if (m.get("s3") != null) {
Map<String, String> ms3 = (Map) m.get("s3");
String accessKey = ms3.get("access_key_id");
String secretAccessKey = ms3.get("secret_access_key");
String region = ms3.get("region");
s3_bucket = ms3.get("bucket");
s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(s3_bucket));
try {
processDirectory(s3, listing, s3_bucket);
} catch (Exception error) {
System.err.println("ERROR IN AWS LISTING: " + error.toString());
}
}
if (m.get("ssl") != null) {
Map x = (Map) m.get("ssl");
ssl = new SSL();
ssl.setKeyManagerPassword = (String) x.get("setKeyManagerPassword");
ssl.setKeyStorePassword = (String) x.get("setKeyStorePassword");
ssl.setKeyStorePath = (String) x.get("setKeyStorePath");
}
/**
* Create the seats id map, and create the bin and win handler classes
* for each exchange
*/
seatsList = (List<Map>) m.get("seats");
for (int i = 0; i < seatsList.size(); i++) {
Map x = seatsList.get(i);
String seatId = (String) x.get("id");
String className = (String) x.get("bid");
int k = className.indexOf("=");
String[] parts = new String[2];
String uri = className.substring(0, k);
className = className.substring(k + 1);
String[] options = null;
/**
* set up any options on the class string
*/
if (className.contains("&")) {
parts = className.split("&");
className = parts[0].trim();
options = parts[1].split(",");
for (int ind = 0; ind < options.length; ind++) {
options[ind] = options[ind].trim();
}
}
String[] tags = uri.split("/");
String exchange = tags[tags.length - 1];
String name = (String) x.get("name");
if (name == null)
name = exchange;
String id = (String) x.get("id");
seats.put(name, id);
try {
Class<?> c = Class.forName(className);
BidRequest br = (BidRequest) c.newInstance();
if (br == null) {
throw new Exception("Could not make new instance of: " + className);
}
Map extension = (Map) x.get("extension");
if (x != null)
br.handleConfigExtensions(extension);
/**
* Handle generic-ized exchanges
*/
if (className.contains("Generic")) {
br.setExchange(exchange);
br.usesEncodedAdm = true;
}
RTBServer.exchanges.put(uri, br);
if (parts[0] != null) {
for (int ind = 1; ind < parts.length; ind++) {
String option = parts[ind];
String[] tuples = option.split("=");
switch(tuples[0]) {
case "usesEncodedAdm":
br.usesEncodedAdm = true;
break;
case "!usesEncodedAdm":
br.usesEncodedAdm = false;
break;
case "rlog":
Double rlog = Double.parseDouble(tuples[1]);
ExchangeLogLevel.getInstance().setExchangeLogLevel(name, rlog.intValue());
break;
case "useStrings":
break;
case "!useStrings":
break;
default:
System.err.println("Unknown request: " + tuples[0] + " in definition of " + className);
}
}
}
/**
* Appnexus requires additional support for ready, pixel and
* click
*/
if (className.contains("Appnexus")) {
RTBServer.exchanges.put(uri + "/ready", new Appnexus(Appnexus.READY));
RTBServer.exchanges.put(uri + "/pixel", new Appnexus(Appnexus.PIXEL));
RTBServer.exchanges.put(uri + "/click", new Appnexus(Appnexus.CLICK));
RTBServer.exchanges.put(uri + "/delivered", new Appnexus(Appnexus.DELIVERED));
Appnexus.seatId = seatId;
}
} catch (Exception error) {
System.err.println("Error configuring exchange: " + name + ", error = ");
throw error;
}
}
/**
* Create forensiq
*/
Map fraud = (Map) m.get("fraud");
if (fraud != null) {
if (m.get("forensiq") != null) {
System.out.println("*** Fraud detection is set to Forensiq");
Map f = (Map) m.get("forensiq");
String ck = (String) f.get("ck");
Integer x = (Integer) f.get("threshhold");
if (!(x == 0 || ck == null || ck.equals("none"))) {
ForensiqClient fx = ForensiqClient.build(ck);
if (fraud.get("endpoint") != null) {
fx.endpoint = (String) fraud.get("endpoint");
}
if (fraud.get("bidOnError") != null) {
fx.bidOnError = (Boolean) fraud.get("bidOnError");
}
if (f.get("connections") != null)
ForensiqClient.getInstance().connections = (int) (Integer) fraud.get("connections");
forensiq = fx;
}
} else {
System.out.println("*** Fraud detection is set to MMDB");
String db = (String) fraud.get("db");
MMDBClient fy = MMDBClient.build(db);
if (fraud.get("bidOnError") != null) {
fy.bidOnError = (Boolean) fraud.get("bidOnError");
}
if (fraud.get("watchlist") != null) {
fy.setWatchlist((List<String>) fraud.get("watchlist"));
}
forensiq = fy;
}
} else {
System.out.println("*** NO Fraud detection");
}
/**
* Deal with the app object
*/
m = (Map) m.get("app");
password = (String) m.get("password");
if (m.get("threads") != null) {
RTBServer.threads = (Integer) m.get("threads");
}
if (m.get("multibid") != null) {
multibid = (Boolean) m.get("multibid");
}
if (m.get("adminPort") != null) {
adminPort = (Integer) m.get("adminPort");
}
if (m.get("adminSSL") != null) {
adminSSL = (Boolean) m.get("adminSSL");
}
String strategy = (String) m.get("strategy");
if (strategy != null && strategy.equals("heuristic"))
RTBServer.strategy = STRATEGY_HEURISTIC;
else
RTBServer.strategy = STRATEGY_MAX_CONNECTIONS;
verbosity = (Map) m.get("verbosity");
if (verbosity != null) {
logLevel = (Integer) verbosity.get("level");
printNoBidReason = (Boolean) verbosity.get("nobid-reason");
}
template = (Map) m.get("template");
if (template == null) {
throw new Exception("No template defined");
}
encodeTemplates();
encodeTemplateStubs();
geotags = (Map) m.get("geotags");
if (geotags != null) {
String states = (String) geotags.get("states");
String codes = (String) geotags.get("zipcodes");
geoTagger.initTags(states, codes);
}
Boolean bValue = false;
bValue = (Boolean) m.get("stopped");
if (bValue != null && bValue == true) {
RTBServer.stopped = true;
pauseOnStart = true;
}
Map redis = (Map) m.get("redis");
if (redis != null) {
Integer rsize = (Integer) redis.get("pool");
if (rsize == null)
rsize = 64;
String host = (String) redis.get("host");
Integer rport = (Integer) redis.get("port");
if (rport == null)
rport = 6379;
// JedisPoolConfig poolConfig = new JedisPoolConfig();;
// configJedis.setMaxTotal(rsize);
// configJedis.setMaxWaitMillis(10);
// poolConfig.setMaxIdle(4000);
// Tests whether connections are dead during idle periods
// poolConfig.setTestWhileIdle(true);
// poolConfig.setMaxTotal(4000);
// poolConfig.setMaxWaitMillis(30);
// jedisPool = new JedisPool(poolConfig,host,rport);
MyJedisPool.host = host;
MyJedisPool.port = rport;
jedisPool = new MyJedisPool(1000, 1000, 5);
System.out.println("*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + rport + ", size = " + rsize);
}
Map zeromq = (Map) m.get("zeromq");
String value = null;
Double dValue = 0.0;
bValue = false;
Map r = (Map) m.get("aerospike");
if (r != null) {
if ((value = (String) r.get("host")) != null)
cacheHost = value;
if (r.get("port") != null)
cachePort = (Integer) r.get("port");
if (r.get("maxconns") != null)
maxconns = (Integer) r.get("maxconns");
AerospikeHandler.getInstance(cacheHost, cachePort, maxconns);
redisson = new RedissonClient(AerospikeHandler.getInstance());
Database.getInstance(redisson);
System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + maxconns + ", handlers: " + AerospikeHandler.getInstance().getCount() + " ***");
String key = (String) m.get("deadmanswitch");
if (key != null) {
deadmanSwitch = new DeadmanSwitch(redisson, key);
}
} else {
redisson = new RedissonClient();
Database db = Database.getInstance(redisson);
readDatabaseIntoCache("database.json");
readBlackListIntoCache("blacklist.json");
}
/**
* Zeromq
*/
if ((value = (String) zeromq.get("bidchannel")) != null)
BIDS_CHANNEL = value;
if ((value = (String) zeromq.get("nobidchannel")) != null)
NOBIDS_CHANNEL = value;
if ((value = (String) zeromq.get("winchannel")) != null)
WINS_CHANNEL = value;
if ((value = (String) zeromq.get("requests")) != null)
REQUEST_CHANNEL = value;
if ((value = (String) zeromq.get("unilogger")) != null)
UNILOGGER_CHANNEL = value;
if ((value = (String) zeromq.get("logger")) != null)
LOG_CHANNEL = value;
if ((value = (String) zeromq.get("clicks")) != null)
CLICKS_CHANNEL = value;
if ((value = (String) zeromq.get("fraud")) != null)
FORENSIQ_CHANNEL = value;
if ((value = (String) zeromq.get("responses")) != null)
RESPONSES = value;
if ((value = (String) zeromq.get("status")) != null)
PERF_CHANNEL = value;
if ((value = (String) zeromq.get("reasons")) != null)
REASONS_CHANNEL = value;
Map xx = (Map) zeromq.get("subscribers");
List<String> list = (List) xx.get("hosts");
commandsPort = (String) xx.get("commands");
for (String host : list) {
String address = "tcp://" + host + ":" + commandsPort + "&commands";
commandAddresses.add(address);
}
if (zeromq.get("requeststrategy") != null) {
Object obj = zeromq.get("requeststrategy");
if (obj instanceof String) {
strategy = (String) zeromq.get("requeststrategy");
if (strategy.equalsIgnoreCase("all") || strategy.equalsIgnoreCase("requests"))
requstLogStrategy = REQUEST_STRATEGY_ALL;
if (strategy.equalsIgnoreCase("bids"))
requstLogStrategy = REQUEST_STRATEGY_BIDS;
if (strategy.equalsIgnoreCase("WINS"))
requstLogStrategy = REQUEST_STRATEGY_WINS;
} else {
if (obj instanceof Integer) {
int level = (Integer) obj;
ExchangeLogLevel.getInstance().setStdLevel(level);
} else if (obj instanceof Double) {
Double perc = (Double) obj;
ExchangeLogLevel.getInstance().setStdLevel(perc.intValue());
}
}
}
/********************************************************************/
campaignsList.clear();
pixelTrackingUrl = (String) m.get("pixel-tracking-url");
winUrl = (String) m.get("winurl");
redirectUrl = (String) m.get("redirect-url");
if (m.get("ttl") != null) {
ttl = (Integer) m.get("ttl");
}
initialLoadlist = (List<Map>) m.get("campaigns");
for (Map<String, String> camp : initialLoadlist) {
if (camp.get("id") != null) {
addCampaign(camp.get("name"), camp.get("id"));
} else {
Controller.getInstance().sendLog(1, "Configuration", "*** ERRORS DETECTED IN INITIAL LOAD OF CAMPAIGNS *** ");
}
}
if (cacheHost == null)
Controller.getInstance().sendLog(1, "Configuration", "*** NO AEROSPIKE CONFIGURED, USING CACH2K INSTEAD *** ");
if (winUrl.contains("localhost")) {
Controller.getInstance().sendLog(1, "Configuration", "*** WIN URL IS SET TO LOCALHOST, NO REMOTE ACCESS WILL WORK FOR WINS ***");
}
}
use of com.amazonaws.services.s3.model.ListObjectsRequest in project zeppelin by apache.
the class S3NotebookRepo method remove.
@Override
public void remove(String noteId, AuthenticationInfo subject) throws IOException {
String key = user + "/" + "notebook" + "/" + noteId;
final ListObjectsRequest listObjectsRequest = new ListObjectsRequest().withBucketName(bucketName).withPrefix(key);
try {
ObjectListing objects = s3client.listObjects(listObjectsRequest);
do {
for (S3ObjectSummary objectSummary : objects.getObjectSummaries()) {
s3client.deleteObject(bucketName, objectSummary.getKey());
}
objects = s3client.listNextBatchOfObjects(objects);
} while (objects.isTruncated());
} catch (AmazonClientException ace) {
throw new IOException("Unable to remove note in S3: " + ace, ace);
}
}
use of com.amazonaws.services.s3.model.ListObjectsRequest in project elasticsearch by elastic.
the class MockAmazonS3 method listObjects.
@Override
public ObjectListing listObjects(ListObjectsRequest listObjectsRequest) throws AmazonClientException, AmazonServiceException {
MockObjectListing list = new MockObjectListing();
list.setTruncated(false);
String blobName;
String prefix = listObjectsRequest.getPrefix();
ArrayList<S3ObjectSummary> mockObjectSummaries = new ArrayList<>();
for (Map.Entry<String, InputStream> blob : blobs.entrySet()) {
blobName = blob.getKey();
S3ObjectSummary objectSummary = new S3ObjectSummary();
if (prefix.isEmpty() || blobName.startsWith(prefix)) {
objectSummary.setKey(blobName);
try {
objectSummary.setSize(getSize(blob.getValue()));
} catch (IOException e) {
throw new AmazonS3Exception("Object listing " + "failed for blob [" + blob.getKey() + "]");
}
mockObjectSummaries.add(objectSummary);
}
}
list.setObjectSummaries(mockObjectSummaries);
return list;
}
use of com.amazonaws.services.s3.model.ListObjectsRequest in project camel by apache.
the class S3Consumer method poll.
@Override
protected int poll() throws Exception {
// must reset for each poll
shutdownRunningTask = null;
pendingExchanges = 0;
String fileName = getConfiguration().getFileName();
String bucketName = getConfiguration().getBucketName();
Queue<Exchange> exchanges;
if (fileName != null) {
LOG.trace("Getting object in bucket [{}] with file name [{}]...", bucketName, fileName);
S3Object s3Object = getAmazonS3Client().getObject(new GetObjectRequest(bucketName, fileName));
exchanges = createExchanges(s3Object);
} else {
LOG.trace("Queueing objects in bucket [{}]...", bucketName);
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName(bucketName);
listObjectsRequest.setPrefix(getConfiguration().getPrefix());
if (maxMessagesPerPoll > 0) {
listObjectsRequest.setMaxKeys(maxMessagesPerPoll);
}
// if there was a marker from previous poll then use that to continue from where we left last time
if (marker != null) {
LOG.trace("Resuming from marker: {}", marker);
listObjectsRequest.setMarker(marker);
}
ObjectListing listObjects = getAmazonS3Client().listObjects(listObjectsRequest);
if (listObjects.isTruncated()) {
marker = listObjects.getNextMarker();
LOG.trace("Returned list is truncated, so setting next marker: {}", marker);
} else {
// no more data so clear marker
marker = null;
}
if (LOG.isTraceEnabled()) {
LOG.trace("Found {} objects in bucket [{}]...", listObjects.getObjectSummaries().size(), bucketName);
}
exchanges = createExchanges(listObjects.getObjectSummaries());
}
return processBatch(CastUtils.cast(exchanges));
}
Aggregations