Search in sources :

Example 6 with Region

use of com.amazonaws.services.s3.model.Region in project jackrabbit-oak by apache.

the class S3Backend method init.

public void init(CachingDataStore store, String homeDir, Properties prop) throws DataStoreException {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        startTime = new Date();
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        LOG.debug("init");
        this.store = store;
        s3ReqDecorator = new S3RequestDecorator(prop);
        s3service = Utils.openService(prop);
        if (bucket == null || "".equals(bucket.trim())) {
            bucket = prop.getProperty(S3Constants.S3_BUCKET);
            // Alternately check if the 'container' property is set
            if (Strings.isNullOrEmpty(bucket)) {
                bucket = prop.getProperty(S3Constants.S3_CONTAINER);
            }
        }
        String region = prop.getProperty(S3Constants.S3_REGION);
        Region s3Region = null;
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
            if (ec2Region != null) {
                s3Region = Region.fromValue(ec2Region.getName());
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
            }
        } else {
            if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                s3Region = Region.US_Standard;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                s3Region = Region.EU_Ireland;
            } else {
                s3Region = Region.fromValue(region);
            }
        }
        if (!s3service.doesBucketExist(bucket)) {
            s3service.createBucket(bucket, s3Region);
            LOG.info("Created bucket [{}] in [{}] ", bucket, region);
        } else {
            LOG.info("Using bucket [{}] in [{}] ", bucket, region);
        }
        int writeThreads = 10;
        String writeThreadsStr = prop.getProperty(S3Constants.S3_WRITE_THREADS);
        if (writeThreadsStr != null) {
            writeThreads = Integer.parseInt(writeThreadsStr);
        }
        LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
        tmx = new TransferManager(s3service, (ThreadPoolExecutor) Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
        int asyncWritePoolSize = 10;
        String maxConnsStr = prop.getProperty(S3Constants.S3_MAX_CONNS);
        if (maxConnsStr != null) {
            asyncWritePoolSize = Integer.parseInt(maxConnsStr) - writeThreads;
        }
        asyncWriteExecuter = (ThreadPoolExecutor) Executors.newFixedThreadPool(asyncWritePoolSize, new NamedThreadFactory("s3-write-worker"));
        String renameKeyProp = prop.getProperty(S3Constants.S3_RENAME_KEYS);
        boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
        LOG.info("Rename keys [{}]", renameKeyBool);
        if (renameKeyBool) {
            renameKeys();
        }
        LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
    } catch (Exception e) {
        LOG.debug("  error ", e);
        Map<String, String> filteredMap = Maps.newHashMap();
        if (prop != null) {
            filteredMap = Maps.filterKeys(Maps.fromProperties(prop), new Predicate<String>() {

                @Override
                public boolean apply(String input) {
                    return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
                }
            });
        }
        throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}
Also used : TransferManager(com.amazonaws.services.s3.transfer.TransferManager) DataStoreException(org.apache.jackrabbit.core.data.DataStoreException) NamedThreadFactory(org.apache.jackrabbit.core.data.util.NamedThreadFactory) AmazonClientException(com.amazonaws.AmazonClientException) Date(java.util.Date) AmazonServiceException(com.amazonaws.AmazonServiceException) AmazonClientException(com.amazonaws.AmazonClientException) DataStoreException(org.apache.jackrabbit.core.data.DataStoreException) IOException(java.io.IOException) S3RequestDecorator(org.apache.jackrabbit.oak.blob.cloud.s3.S3RequestDecorator) Region(com.amazonaws.services.s3.model.Region) ThreadPoolExecutor(java.util.concurrent.ThreadPoolExecutor) Map(java.util.Map)

Example 7 with Region

use of com.amazonaws.services.s3.model.Region in project jackrabbit-oak by apache.

the class S3Backend method init.

public void init() throws DataStoreException {
    ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
    try {
        startTime = new Date();
        Thread.currentThread().setContextClassLoader(getClass().getClassLoader());
        LOG.debug("init");
        s3ReqDecorator = new S3RequestDecorator(properties);
        s3service = Utils.openService(properties);
        if (bucket == null || "".equals(bucket.trim())) {
            bucket = properties.getProperty(S3Constants.S3_BUCKET);
            // Alternately check if the 'container' property is set
            if (Strings.isNullOrEmpty(bucket)) {
                bucket = properties.getProperty(S3Constants.S3_CONTAINER);
            }
        }
        secret = properties.getProperty("secret");
        String region = properties.getProperty(S3Constants.S3_REGION);
        Region s3Region;
        if (StringUtils.isNullOrEmpty(region)) {
            com.amazonaws.regions.Region ec2Region = Regions.getCurrentRegion();
            if (ec2Region != null) {
                s3Region = Region.fromValue(ec2Region.getName());
            } else {
                throw new AmazonClientException("parameter [" + S3Constants.S3_REGION + "] not configured and cannot be derived from environment");
            }
        } else {
            if (Utils.DEFAULT_AWS_BUCKET_REGION.equals(region)) {
                s3Region = Region.US_Standard;
            } else if (Region.EU_Ireland.toString().equals(region)) {
                s3Region = Region.EU_Ireland;
            } else {
                s3Region = Region.fromValue(region);
            }
        }
        if (!s3service.doesBucketExist(bucket)) {
            s3service.createBucket(bucket, s3Region);
            LOG.info("Created bucket [{}] in [{}] ", bucket, region);
        } else {
            LOG.info("Using bucket [{}] in [{}] ", bucket, region);
        }
        int writeThreads = 10;
        String writeThreadsStr = properties.getProperty(S3Constants.S3_WRITE_THREADS);
        if (writeThreadsStr != null) {
            writeThreads = Integer.parseInt(writeThreadsStr);
        }
        LOG.info("Using thread pool of [{}] threads in S3 transfer manager.", writeThreads);
        tmx = new TransferManager(s3service, Executors.newFixedThreadPool(writeThreads, new NamedThreadFactory("s3-transfer-manager-worker")));
        String renameKeyProp = properties.getProperty(S3Constants.S3_RENAME_KEYS);
        boolean renameKeyBool = (renameKeyProp == null || "".equals(renameKeyProp)) ? false : Boolean.parseBoolean(renameKeyProp);
        LOG.info("Rename keys [{}]", renameKeyBool);
        if (renameKeyBool) {
            renameKeys();
        }
        LOG.debug("S3 Backend initialized in [{}] ms", +(System.currentTimeMillis() - startTime.getTime()));
    } catch (Exception e) {
        LOG.debug("  error ", e);
        Map<String, String> filteredMap = Maps.newHashMap();
        if (properties != null) {
            filteredMap = Maps.filterKeys(Maps.fromProperties(properties), new Predicate<String>() {

                @Override
                public boolean apply(String input) {
                    return !input.equals(S3Constants.ACCESS_KEY) && !input.equals(S3Constants.SECRET_KEY);
                }
            });
        }
        throw new DataStoreException("Could not initialize S3 from " + filteredMap, e);
    } finally {
        if (contextClassLoader != null) {
            Thread.currentThread().setContextClassLoader(contextClassLoader);
        }
    }
}
Also used : TransferManager(com.amazonaws.services.s3.transfer.TransferManager) DataStoreException(org.apache.jackrabbit.core.data.DataStoreException) NamedThreadFactory(org.apache.jackrabbit.core.data.util.NamedThreadFactory) AmazonClientException(com.amazonaws.AmazonClientException) Date(java.util.Date) DataStoreException(org.apache.jackrabbit.core.data.DataStoreException) AmazonServiceException(com.amazonaws.AmazonServiceException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) AmazonClientException(com.amazonaws.AmazonClientException) Region(com.amazonaws.services.s3.model.Region) Map(java.util.Map)

Example 8 with Region

use of com.amazonaws.services.s3.model.Region in project XRTB by benmfaul.

the class Configuration method initialize.

/**
	 * Read the Java Bean Shell file that initializes this constructor.
	 * 
	 * @param path.
	 *            String - The file name containing the Java Bean Shell code.
	 * @throws Exception
	 *             on file errors.
	 */
public void initialize(String path, String shard, int port, int sslPort) throws Exception {
    this.fileName = path;
    /******************************
		 * System Name
		 *****************************/
    this.shard = shard;
    this.port = port;
    this.sslPort = sslPort;
    java.net.InetAddress localMachine = null;
    String useName = null;
    try {
        localMachine = java.net.InetAddress.getLocalHost();
        ipAddress = localMachine.getHostAddress();
        useName = localMachine.getHostName();
    } catch (Exception error) {
        useName = getIpAddress();
    }
    if (shard == null || shard.length() == 0)
        instanceName = useName + ":" + port;
    else
        instanceName = shard + ":" + useName + ":" + port;
    /**
		 * Set up tem p files
		 */
    // create the temp
    Files.createDirectories(Paths.get("www/temp"));
    // directory in www so
    // preview campaign will
    // work
    /*********************************************
		 * USE ZOOKEEPER, AEROSPIKE OR FILE CONFIG
		 *********************************************/
    String str = null;
    if (path.startsWith("zookeeper")) {
        String[] parts = path.split(":");
        System.out.println(parts);
        zk = new ZkConnect(parts[1]);
        zk.join(parts[2], "bidders", instanceName);
        str = zk.readConfig(parts[2] + "/bidders");
    } else if (path.startsWith("aerospike")) {
        String[] parts = path.split(":");
        System.out.println(parts);
        String aerospike = parts[1];
        String configKey = parts[2];
        AerospikeHandler spike = AerospikeHandler.getInstance(aerospike, 3000, 300);
        redisson = new RedissonClient(spike);
        Database.getInstance(redisson);
        str = redisson.get(configKey);
        if (str == null) {
            throw new Exception("Aerospike configuration at " + path + " not available.");
        }
        System.out.println(str);
    } else {
        byte[] encoded = Files.readAllBytes(Paths.get(path));
        str = Charset.defaultCharset().decode(ByteBuffer.wrap(encoded)).toString();
    }
    Map<?, ?> m = DbTools.mapper.readValue(str, Map.class);
    /*******************************************************************************/
    seats = new HashMap<String, String>();
    if (m.get("lists") != null) {
        filesList = (List) m.get("lists");
        initializeLookingGlass(filesList);
    }
    if (m.get("s3") != null) {
        Map<String, String> ms3 = (Map) m.get("s3");
        String accessKey = ms3.get("access_key_id");
        String secretAccessKey = ms3.get("secret_access_key");
        String region = ms3.get("region");
        s3_bucket = ms3.get("bucket");
        s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretAccessKey));
        ObjectListing listing = s3.listObjects(new ListObjectsRequest().withBucketName(s3_bucket));
        try {
            processDirectory(s3, listing, s3_bucket);
        } catch (Exception error) {
            System.err.println("ERROR IN AWS LISTING: " + error.toString());
        }
    }
    if (m.get("ssl") != null) {
        Map x = (Map) m.get("ssl");
        ssl = new SSL();
        ssl.setKeyManagerPassword = (String) x.get("setKeyManagerPassword");
        ssl.setKeyStorePassword = (String) x.get("setKeyStorePassword");
        ssl.setKeyStorePath = (String) x.get("setKeyStorePath");
    }
    /**
		 * Create the seats id map, and create the bin and win handler classes
		 * for each exchange
		 */
    seatsList = (List<Map>) m.get("seats");
    for (int i = 0; i < seatsList.size(); i++) {
        Map x = seatsList.get(i);
        String seatId = (String) x.get("id");
        String className = (String) x.get("bid");
        int k = className.indexOf("=");
        String[] parts = new String[2];
        String uri = className.substring(0, k);
        className = className.substring(k + 1);
        String[] options = null;
        /**
			 * set up any options on the class string
			 */
        if (className.contains("&")) {
            parts = className.split("&");
            className = parts[0].trim();
            options = parts[1].split(",");
            for (int ind = 0; ind < options.length; ind++) {
                options[ind] = options[ind].trim();
            }
        }
        String[] tags = uri.split("/");
        String exchange = tags[tags.length - 1];
        String name = (String) x.get("name");
        if (name == null)
            name = exchange;
        String id = (String) x.get("id");
        seats.put(name, id);
        try {
            Class<?> c = Class.forName(className);
            BidRequest br = (BidRequest) c.newInstance();
            if (br == null) {
                throw new Exception("Could not make new instance of: " + className);
            }
            Map extension = (Map) x.get("extension");
            if (x != null)
                br.handleConfigExtensions(extension);
            /**
				 * Handle generic-ized exchanges
				 */
            if (className.contains("Generic")) {
                br.setExchange(exchange);
                br.usesEncodedAdm = true;
            }
            RTBServer.exchanges.put(uri, br);
            if (parts[0] != null) {
                for (int ind = 1; ind < parts.length; ind++) {
                    String option = parts[ind];
                    String[] tuples = option.split("=");
                    switch(tuples[0]) {
                        case "usesEncodedAdm":
                            br.usesEncodedAdm = true;
                            break;
                        case "!usesEncodedAdm":
                            br.usesEncodedAdm = false;
                            break;
                        case "rlog":
                            Double rlog = Double.parseDouble(tuples[1]);
                            ExchangeLogLevel.getInstance().setExchangeLogLevel(name, rlog.intValue());
                            break;
                        case "useStrings":
                            break;
                        case "!useStrings":
                            break;
                        default:
                            System.err.println("Unknown request: " + tuples[0] + " in definition of " + className);
                    }
                }
            }
            /**
				 * Appnexus requires additional support for ready, pixel and
				 * click
				 */
            if (className.contains("Appnexus")) {
                RTBServer.exchanges.put(uri + "/ready", new Appnexus(Appnexus.READY));
                RTBServer.exchanges.put(uri + "/pixel", new Appnexus(Appnexus.PIXEL));
                RTBServer.exchanges.put(uri + "/click", new Appnexus(Appnexus.CLICK));
                RTBServer.exchanges.put(uri + "/delivered", new Appnexus(Appnexus.DELIVERED));
                Appnexus.seatId = seatId;
            }
        } catch (Exception error) {
            System.err.println("Error configuring exchange: " + name + ", error = ");
            throw error;
        }
    }
    /**
		 * Create forensiq
		 */
    Map fraud = (Map) m.get("fraud");
    if (fraud != null) {
        if (m.get("forensiq") != null) {
            System.out.println("*** Fraud detection is set to Forensiq");
            Map f = (Map) m.get("forensiq");
            String ck = (String) f.get("ck");
            Integer x = (Integer) f.get("threshhold");
            if (!(x == 0 || ck == null || ck.equals("none"))) {
                ForensiqClient fx = ForensiqClient.build(ck);
                if (fraud.get("endpoint") != null) {
                    fx.endpoint = (String) fraud.get("endpoint");
                }
                if (fraud.get("bidOnError") != null) {
                    fx.bidOnError = (Boolean) fraud.get("bidOnError");
                }
                if (f.get("connections") != null)
                    ForensiqClient.getInstance().connections = (int) (Integer) fraud.get("connections");
                forensiq = fx;
            }
        } else {
            System.out.println("*** Fraud detection is set to MMDB");
            String db = (String) fraud.get("db");
            MMDBClient fy = MMDBClient.build(db);
            if (fraud.get("bidOnError") != null) {
                fy.bidOnError = (Boolean) fraud.get("bidOnError");
            }
            if (fraud.get("watchlist") != null) {
                fy.setWatchlist((List<String>) fraud.get("watchlist"));
            }
            forensiq = fy;
        }
    } else {
        System.out.println("*** NO Fraud detection");
    }
    /**
		 * Deal with the app object
		 */
    m = (Map) m.get("app");
    password = (String) m.get("password");
    if (m.get("threads") != null) {
        RTBServer.threads = (Integer) m.get("threads");
    }
    if (m.get("multibid") != null) {
        multibid = (Boolean) m.get("multibid");
    }
    if (m.get("adminPort") != null) {
        adminPort = (Integer) m.get("adminPort");
    }
    if (m.get("adminSSL") != null) {
        adminSSL = (Boolean) m.get("adminSSL");
    }
    String strategy = (String) m.get("strategy");
    if (strategy != null && strategy.equals("heuristic"))
        RTBServer.strategy = STRATEGY_HEURISTIC;
    else
        RTBServer.strategy = STRATEGY_MAX_CONNECTIONS;
    verbosity = (Map) m.get("verbosity");
    if (verbosity != null) {
        logLevel = (Integer) verbosity.get("level");
        printNoBidReason = (Boolean) verbosity.get("nobid-reason");
    }
    template = (Map) m.get("template");
    if (template == null) {
        throw new Exception("No template defined");
    }
    encodeTemplates();
    encodeTemplateStubs();
    geotags = (Map) m.get("geotags");
    if (geotags != null) {
        String states = (String) geotags.get("states");
        String codes = (String) geotags.get("zipcodes");
        geoTagger.initTags(states, codes);
    }
    Boolean bValue = false;
    bValue = (Boolean) m.get("stopped");
    if (bValue != null && bValue == true) {
        RTBServer.stopped = true;
        pauseOnStart = true;
    }
    Map redis = (Map) m.get("redis");
    if (redis != null) {
        Integer rsize = (Integer) redis.get("pool");
        if (rsize == null)
            rsize = 64;
        String host = (String) redis.get("host");
        Integer rport = (Integer) redis.get("port");
        if (rport == null)
            rport = 6379;
        // JedisPoolConfig poolConfig = new JedisPoolConfig();;
        // configJedis.setMaxTotal(rsize);
        // configJedis.setMaxWaitMillis(10);
        // poolConfig.setMaxIdle(4000);
        // Tests whether connections are dead during idle periods
        // poolConfig.setTestWhileIdle(true);
        // poolConfig.setMaxTotal(4000);
        // poolConfig.setMaxWaitMillis(30);
        // jedisPool = new JedisPool(poolConfig,host,rport);
        MyJedisPool.host = host;
        MyJedisPool.port = rport;
        jedisPool = new MyJedisPool(1000, 1000, 5);
        System.out.println("*** JEDISPOOL = " + jedisPool + ". host = " + host + ", port = " + rport + ", size = " + rsize);
    }
    Map zeromq = (Map) m.get("zeromq");
    String value = null;
    Double dValue = 0.0;
    bValue = false;
    Map r = (Map) m.get("aerospike");
    if (r != null) {
        if ((value = (String) r.get("host")) != null)
            cacheHost = value;
        if (r.get("port") != null)
            cachePort = (Integer) r.get("port");
        if (r.get("maxconns") != null)
            maxconns = (Integer) r.get("maxconns");
        AerospikeHandler.getInstance(cacheHost, cachePort, maxconns);
        redisson = new RedissonClient(AerospikeHandler.getInstance());
        Database.getInstance(redisson);
        System.out.println("*** Aerospike connection set to: " + cacheHost + ":" + cachePort + ", connections = " + maxconns + ", handlers: " + AerospikeHandler.getInstance().getCount() + " ***");
        String key = (String) m.get("deadmanswitch");
        if (key != null) {
            deadmanSwitch = new DeadmanSwitch(redisson, key);
        }
    } else {
        redisson = new RedissonClient();
        Database db = Database.getInstance(redisson);
        readDatabaseIntoCache("database.json");
        readBlackListIntoCache("blacklist.json");
    }
    /**
		 * Zeromq
		 */
    if ((value = (String) zeromq.get("bidchannel")) != null)
        BIDS_CHANNEL = value;
    if ((value = (String) zeromq.get("nobidchannel")) != null)
        NOBIDS_CHANNEL = value;
    if ((value = (String) zeromq.get("winchannel")) != null)
        WINS_CHANNEL = value;
    if ((value = (String) zeromq.get("requests")) != null)
        REQUEST_CHANNEL = value;
    if ((value = (String) zeromq.get("unilogger")) != null)
        UNILOGGER_CHANNEL = value;
    if ((value = (String) zeromq.get("logger")) != null)
        LOG_CHANNEL = value;
    if ((value = (String) zeromq.get("clicks")) != null)
        CLICKS_CHANNEL = value;
    if ((value = (String) zeromq.get("fraud")) != null)
        FORENSIQ_CHANNEL = value;
    if ((value = (String) zeromq.get("responses")) != null)
        RESPONSES = value;
    if ((value = (String) zeromq.get("status")) != null)
        PERF_CHANNEL = value;
    if ((value = (String) zeromq.get("reasons")) != null)
        REASONS_CHANNEL = value;
    Map xx = (Map) zeromq.get("subscribers");
    List<String> list = (List) xx.get("hosts");
    commandsPort = (String) xx.get("commands");
    for (String host : list) {
        String address = "tcp://" + host + ":" + commandsPort + "&commands";
        commandAddresses.add(address);
    }
    if (zeromq.get("requeststrategy") != null) {
        Object obj = zeromq.get("requeststrategy");
        if (obj instanceof String) {
            strategy = (String) zeromq.get("requeststrategy");
            if (strategy.equalsIgnoreCase("all") || strategy.equalsIgnoreCase("requests"))
                requstLogStrategy = REQUEST_STRATEGY_ALL;
            if (strategy.equalsIgnoreCase("bids"))
                requstLogStrategy = REQUEST_STRATEGY_BIDS;
            if (strategy.equalsIgnoreCase("WINS"))
                requstLogStrategy = REQUEST_STRATEGY_WINS;
        } else {
            if (obj instanceof Integer) {
                int level = (Integer) obj;
                ExchangeLogLevel.getInstance().setStdLevel(level);
            } else if (obj instanceof Double) {
                Double perc = (Double) obj;
                ExchangeLogLevel.getInstance().setStdLevel(perc.intValue());
            }
        }
    }
    /********************************************************************/
    campaignsList.clear();
    pixelTrackingUrl = (String) m.get("pixel-tracking-url");
    winUrl = (String) m.get("winurl");
    redirectUrl = (String) m.get("redirect-url");
    if (m.get("ttl") != null) {
        ttl = (Integer) m.get("ttl");
    }
    initialLoadlist = (List<Map>) m.get("campaigns");
    for (Map<String, String> camp : initialLoadlist) {
        if (camp.get("id") != null) {
            addCampaign(camp.get("name"), camp.get("id"));
        } else {
            Controller.getInstance().sendLog(1, "Configuration", "*** ERRORS DETECTED IN INITIAL LOAD OF CAMPAIGNS *** ");
        }
    }
    if (cacheHost == null)
        Controller.getInstance().sendLog(1, "Configuration", "*** NO AEROSPIKE CONFIGURED, USING CACH2K INSTEAD *** ");
    if (winUrl.contains("localhost")) {
        Controller.getInstance().sendLog(1, "Configuration", "*** WIN URL IS SET TO LOCALHOST, NO REMOTE ACCESS WILL WORK FOR WINS ***");
    }
}
Also used : ZkConnect(com.xrtb.tools.ZkConnect) MMDBClient(com.xrtb.fraud.MMDBClient) ObjectListing(com.amazonaws.services.s3.model.ObjectListing) BidRequest(com.xrtb.pojo.BidRequest) ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) DeadmanSwitch(com.xrtb.bidder.DeadmanSwitch) Database(com.xrtb.db.Database) List(java.util.List) ArrayList(java.util.ArrayList) Appnexus(com.xrtb.exchanges.appnexus.Appnexus) BasicAWSCredentials(com.amazonaws.auth.BasicAWSCredentials) AmazonS3Client(com.amazonaws.services.s3.AmazonS3Client) RedissonClient(com.aerospike.redisson.RedissonClient) AerospikeHandler(com.aerospike.redisson.AerospikeHandler) S3Object(com.amazonaws.services.s3.model.S3Object) DataBaseObject(com.xrtb.db.DataBaseObject) Map(java.util.Map) NavMap(com.xrtb.blocks.NavMap) HashMap(java.util.HashMap) ForensiqClient(com.xrtb.fraud.ForensiqClient)

Example 9 with Region

use of com.amazonaws.services.s3.model.Region in project h2o-3 by h2oai.

the class PersistS3 method configureClient.

static AmazonS3Client configureClient(AmazonS3Client s3Client) {
    if (System.getProperty(S3_REGION) != null) {
        String region = System.getProperty(S3_REGION);
        Log.debug("S3 region specified: ", region);
        s3Client.setRegion(RegionUtils.getRegion(region));
    }
    // Region overrides end-point settings
    if (System.getProperty(S3_END_POINT) != null) {
        String endPoint = System.getProperty(S3_END_POINT);
        Log.debug("S3 endpoint specified: ", endPoint);
        s3Client.setEndpoint(endPoint);
    }
    if (System.getProperty(S3_ENABLE_PATH_STYLE) != null && Boolean.valueOf(System.getProperty(S3_ENABLE_PATH_STYLE))) {
        Log.debug("S3 path style access enabled");
        S3ClientOptions sco = new S3ClientOptions();
        sco.setPathStyleAccess(true);
        s3Client.setS3ClientOptions(sco);
    }
    return s3Client;
}
Also used : S3ClientOptions(com.amazonaws.services.s3.S3ClientOptions)

Example 10 with Region

use of com.amazonaws.services.s3.model.Region in project camel by apache.

the class S3Endpoint method doStart.

@Override
public void doStart() throws Exception {
    super.doStart();
    s3Client = configuration.getAmazonS3Client() != null ? configuration.getAmazonS3Client() : createS3Client();
    if (ObjectHelper.isNotEmpty(configuration.getAmazonS3Endpoint())) {
        s3Client.setEndpoint(configuration.getAmazonS3Endpoint());
    }
    String fileName = getConfiguration().getFileName();
    if (fileName != null) {
        LOG.trace("File name [{}] requested, so skipping bucket check...", fileName);
        return;
    }
    String bucketName = getConfiguration().getBucketName();
    LOG.trace("Querying whether bucket [{}] already exists...", bucketName);
    String prefix = getConfiguration().getPrefix();
    try {
        s3Client.listObjects(new ListObjectsRequest(bucketName, prefix, null, null, 0));
        LOG.trace("Bucket [{}] already exists", bucketName);
        return;
    } catch (AmazonServiceException ase) {
        /* 404 means the bucket doesn't exist */
        if (ase.getStatusCode() != 404) {
            throw ase;
        }
    }
    LOG.trace("Bucket [{}] doesn't exist yet", bucketName);
    // creates the new bucket because it doesn't exist yet
    CreateBucketRequest createBucketRequest = new CreateBucketRequest(getConfiguration().getBucketName());
    if (getConfiguration().getRegion() != null) {
        createBucketRequest.setRegion(getConfiguration().getRegion());
    }
    LOG.trace("Creating bucket [{}] in region [{}] with request [{}]...", configuration.getBucketName(), configuration.getRegion(), createBucketRequest);
    s3Client.createBucket(createBucketRequest);
    LOG.trace("Bucket created");
    if (configuration.getPolicy() != null) {
        LOG.trace("Updating bucket [{}] with policy [{}]", bucketName, configuration.getPolicy());
        s3Client.setBucketPolicy(bucketName, configuration.getPolicy());
        LOG.trace("Bucket policy updated");
    }
}
Also used : ListObjectsRequest(com.amazonaws.services.s3.model.ListObjectsRequest) CreateBucketRequest(com.amazonaws.services.s3.model.CreateBucketRequest) AmazonServiceException(com.amazonaws.AmazonServiceException)

Aggregations

AmazonS3Client (com.amazonaws.services.s3.AmazonS3Client)6 AmazonClientException (com.amazonaws.AmazonClientException)5 AmazonServiceException (com.amazonaws.AmazonServiceException)5 ClientConfiguration (com.amazonaws.ClientConfiguration)3 BasicAWSCredentials (com.amazonaws.auth.BasicAWSCredentials)3 Region (com.amazonaws.services.s3.model.Region)3 TransferManager (com.amazonaws.services.s3.transfer.TransferManager)3 IOException (java.io.IOException)3 Date (java.util.Date)3 Map (java.util.Map)3 DataStoreException (org.apache.jackrabbit.core.data.DataStoreException)3 NamedThreadFactory (org.apache.jackrabbit.core.data.util.NamedThreadFactory)3 AWSCredentials (com.amazonaws.auth.AWSCredentials)2 AmazonEC2 (com.amazonaws.services.ec2.AmazonEC2)2 DescribeRegionsResult (com.amazonaws.services.ec2.model.DescribeRegionsResult)2 Region (com.amazonaws.services.ec2.model.Region)2 AmazonS3 (com.amazonaws.services.s3.AmazonS3)2 S3ClientOptions (com.amazonaws.services.s3.S3ClientOptions)2 ListObjectsRequest (com.amazonaws.services.s3.model.ListObjectsRequest)2 ArrayList (java.util.ArrayList)2