Examples with DatarouterS3Client io.datarouter.aws.s3.DatarouterS3Client used on opensource projects

Example 1 with DatarouterS3Client

use of io.datarouter.aws.s3.DatarouterS3Client in project datarouter by hotpads.

the class S3BucketHandler method countObjects.

@Handler
public Mav countObjects(@Param(P_client) String client, @Param(P_bucket) String bucket, @Param(P_prefix) OptionalString prefix) {
    ClientId clientId = clients.getClientId(client);
    DatarouterS3Client s3Client = s3ClientManager.getClient(clientId);
    var count = new AtomicLong();
    var size = new AtomicLong();
    var message = new AtomicReference<String>();
    s3Client.scanObjects(bucket, prefix.orElse("")).each($ -> count.incrementAndGet()).each(obj -> size.addAndGet(obj.size())).sample(10_000, true).map(obj -> String.format("client=%s, bucket=%s, prefix=%s, count=%s, size=%s, through=%s", client, bucket, prefix.orElse(null), NumberFormatter.addCommas(count.get()), NumberFormatter.addCommas(size.get()), obj.key())).each(message::set).forEach(logger::warn);
    return pageFactory.message(request, message.get());
}

Example 2 with DatarouterS3Client

use of io.datarouter.aws.s3.DatarouterS3Client in project datarouter by hotpads.

the class S3BucketHandler method index.

@Handler(defaultHandler = true)
public Mav index(@Param(P_client) String client, @Param(P_bucket) String bucket, @Param(P_prefix) OptionalString prefix, @Param(P_after) OptionalString after, @Param(P_offset) OptionalInteger offset, @Param(P_limit) OptionalInteger limit, @Param(P_currentDirectory) OptionalBoolean currentDirectory, @Param(P_delimiter) OptionalString delimiter) {
    var form = new HtmlForm().withMethod("get");
    form.addTextField().withDisplay("Client").withName(P_client).withPlaceholder("theClientName").withValue(client);
    form.addTextField().withDisplay("Bucket").withName(P_bucket).withPlaceholder("the.bucket.name").withValue(bucket);
    form.addTextField().withDisplay("Prefix").withName(P_prefix).withValue(prefix.orElse(""));
    form.addTextField().withDisplay("After").withName(P_after).withValue(after.orElse(""));
    form.addTextField().withDisplay("Offset").withName(P_offset).withValue(offset.orElse(0) + "");
    form.addTextField().withDisplay("Limit").withName(P_limit).withValue(limit.orElse(100) + "");
    form.addTextField().withDisplay("Delimiter").withName(P_delimiter).withValue(delimiter.orElse(""));
    form.addCheckboxField().withDisplay("currentDirectory").withName(P_currentDirectory).withChecked(currentDirectory.orElse(false));
    form.addButton().withDisplay("Submit").withValue("");
    var htmlForm = Bootstrap4FormHtml.render(form).withClass("card card-body bg-light");
    ClientId clientId = clients.getClientId(client);
    DatarouterS3Client s3Client = s3ClientManager.getClient(clientId);
    List<DirectoryDto> objects = s3Client.scanSubdirectories(bucket, prefix.orElse(null), after.orElse(null), delimiter.orElse(null), limit.orElse(100), currentDirectory.orElse(false)).list();
    int sizePadding = sizePadding(objects);
    ContainerTag<?> table = new J2HtmlTable<DirectoryDto>().withClasses("sortable table table-sm table-striped my-4 border").withHtmlColumn("Key", object -> {
        String name = object.name;
        if (object.isDirectory) {
            return td(makePrefixLink(client, bucket, name, "/"));
        }
        return td(name);
    }).withHtmlColumn("Directory", object -> {
        boolean isDirectory = object.isDirectory;
        if (isDirectory) {
            String href = new URIBuilder().setPath(request.getContextPath() + paths.datarouter.clients.awsS3.countObjects.toSlashedString()).addParameter(P_client, client).addParameter(P_bucket, bucket).addParameter(P_prefix, object.name).toString();
            return td(a("true, view count").withHref(href));
        }
        return td(String.valueOf(isDirectory));
    }).withHtmlColumn("Size", object -> {
        String commas = NumberFormatter.addCommas(object.size);
        String padded = StringTool.pad(commas, ' ', sizePadding);
        String escaped = padded.replaceAll(" ", "&nbsp;");
        return td(rawHtml(escaped));
    }).withColumn("Last Modified", object -> object.lastModified).withColumn("Storage Class", object -> object.storageClass).build(objects);
    ContainerTag<?> tableWrapper = table.withStyle("font-family:monospace; font-size:.9em;");
    var content = div(htmlForm, h4(bucket), tableWrapper).withClass("container-fluid my-4");
    return pageFactory.startBuilder(request).withTitle("S3 Bucket").withRequires(DatarouterWebRequireJsV2.SORTTABLE).withContent(content).buildMav();
}

Example 3 with DatarouterS3Client

use of io.datarouter.aws.s3.DatarouterS3Client in project datarouter by hotpads.

the class S3ClientNodeFactory method createInternal.

/*---------------- private ------------------*/
private S3Node createInternal(NodeParams<PathbeanKey, Pathbean, PathbeanFielder> nodeParams) {
    DatarouterS3Client client = s3ClientManager.getClient(nodeParams.getClientId());
    String bucket = nodeParams.getPhysicalName();
    Subpath path = nodeParams.getPath();
    var s3DirectoryManager = new S3DirectoryManager(client, bucket, path);
    return new S3Node(nodeParams, s3ClientType, client, s3DirectoryManager);
}

Example 4 with DatarouterS3Client

use of io.datarouter.aws.s3.DatarouterS3Client in project datarouter by hotpads.

the class S3ClientManager method safeInitClient.

@Override
protected void safeInitClient(ClientId clientId) {
    if (clientByClientId.containsKey(clientId)) {
        throw new RuntimeException(clientId + " already exists");
    }
    PhaseTimer timer = new PhaseTimer(clientId.getName());
    DatarouterS3Client client = create(clientId);
    clientByClientId.put(clientId, client);
    timer.add("create");
    logger.warn("{}", timer);
}

Example 5 with DatarouterS3Client

use of io.datarouter.aws.s3.DatarouterS3Client in project datarouter by hotpads.

the class S3WebInspector method buildBucketTable.

private ContainerTag<?> buildBucketTable(String contextPath, ClientId clientId) {
    DatarouterS3Client client = s3ClientManager.getClient(clientId);
    List<S3BucketDto> buckets = client.scanBuckets().parallel(new ParallelScannerContext(bucketRegionExecutor, bucketRegionExecutor.getMaximumPoolSize(), true)).map(bucket -> {
        // RPC
        Region region = client.getBucketRegion(bucket.name());
        return new S3BucketDto(clientId.getName(), bucket.name(), region, bucket.creationDate());
    }).sort(Comparator.comparing(bucket -> bucket.bucketName.toLowerCase())).list();
    var table = new J2HtmlTable<S3BucketDto>().withClasses("sortable table table-sm table-striped my-4 border").withHtmlColumn("Name", bucket -> {
        String href = new URIBuilder().setPath(contextPath + paths.datarouter.clients.awsS3.listObjects.toSlashedString()).addParameter(S3BucketHandler.P_client, bucket.clientName).addParameter(S3BucketHandler.P_bucket, bucket.bucketName).addParameter(S3BucketHandler.P_delimiter, "/").toString();
        return td(a(bucket.bucketName).withHref(href));
    }).withColumn("Region", bucket -> bucket.region).withColumn("Created", bucket -> bucket.creationDate).build(buckets);
    return div(h4("Buckets - " + buckets.size()), table).withClass("container-fluid my-4").withStyle("padding-left: 0px");
}