Search in sources :

Example 1 with SnapshotGroup

use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.

the class DatarouterSnapshotEntriesHandler method buildTable.

private DomContent buildTable(SnapshotKey snapshotKey, long offset, long limit) {
    SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
    var reader = new ScanningSnapshotReader(snapshotKey, exec, 2, groups, 1);
    SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
    List<SnapshotRecordStrings> rows = reader.scan(0).skip(offset).limit(limit).map(decoder::decode).list();
    var table = new J2HtmlTable<SnapshotRecordStrings>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("id", row -> row.id).withColumn(decoder.keyName(), row -> row.key).withColumn(decoder.valueName(), row -> {
        if (row.value == null) {
            return "";
        } else if (row.value.length() < 64) {
            return row.value;
        } else {
            return row.value.subSequence(0, 64) + "...";
        }
    }).withHtmlColumn("details", row -> {
        String href = new URIBuilder().setPath(request.getContextPath() + snapshotPaths.datarouter.snapshot.individual.entry.toSlashedString()).addParameter(DatarouterSnapshotEntryHandler.P_groupId, snapshotKey.groupId).addParameter(DatarouterSnapshotEntryHandler.P_snapshotId, snapshotKey.snapshotId).addParameter(DatarouterSnapshotEntryHandler.P_id, Long.toString(row.id)).toString();
        return td(a("view").withHref(href));
    }).build(rows);
    return table;
}
Also used : ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) URIBuilder(org.apache.http.client.utils.URIBuilder) Mav(io.datarouter.web.handler.mav.Mav) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) ReflectionTool(io.datarouter.util.lang.ReflectionTool) TagCreator.a(j2html.TagCreator.a) DatarouterSnapshotWebExecutor(io.datarouter.filesystem.snapshot.web.DatarouterSnapshotExecutors.DatarouterSnapshotWebExecutor) OptionalLong(io.datarouter.web.handler.types.optional.OptionalLong) Inject(javax.inject.Inject) DomContent(j2html.tags.DomContent) List(java.util.List) DatarouterWebRequireJsV2(io.datarouter.web.requirejs.DatarouterWebRequireJsV2) ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) SnapshotGroups(io.datarouter.filesystem.snapshot.group.SnapshotGroups) TagCreator.td(j2html.TagCreator.td) BaseHandler(io.datarouter.web.handler.BaseHandler) Bootstrap4PageFactory(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) Param(io.datarouter.web.handler.types.Param) TagCreator.div(j2html.TagCreator.div) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) URIBuilder(org.apache.http.client.utils.URIBuilder)

Example 2 with SnapshotGroup

use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.

the class DatarouterSnapshotEntryHandler method buildTable.

private ContainerTag<?> buildTable(SnapshotKey snapshotKey, long id) {
    SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
    var reader = new SnapshotIdReader(snapshotKey, groups);
    SnapshotRecord record = reader.getRecord(id);
    SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
    SnapshotRecordStrings decoded = decoder.decode(record);
    List<Twin<String>> rows = new ArrayList<>();
    rows.add(new Twin<>("id", Long.toString(record.id)));
    rows.add(new Twin<>(decoder.keyName(), decoded.key));
    rows.add(new Twin<>(decoder.valueName(), decoded.value));
    IntStream.range(0, decoded.columnValues.size()).mapToObj(column -> new Twin<>(decoder.columnValueName(column), decoded.columnValues.get(column))).forEach(rows::add);
    var table = new J2HtmlTable<Twin<String>>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("field", twin -> twin.getLeft()).withColumn("value", twin -> twin.getRight()).build(rows);
    return table;
}
Also used : IntStream(java.util.stream.IntStream) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) Twin(io.datarouter.util.tuple.Twin) Mav(io.datarouter.web.handler.mav.Mav) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) ReflectionTool(io.datarouter.util.lang.ReflectionTool) ArrayList(java.util.ArrayList) ContainerTag(j2html.tags.ContainerTag) Inject(javax.inject.Inject) List(java.util.List) DatarouterWebRequireJsV2(io.datarouter.web.requirejs.DatarouterWebRequireJsV2) SnapshotGroups(io.datarouter.filesystem.snapshot.group.SnapshotGroups) BaseHandler(io.datarouter.web.handler.BaseHandler) Bootstrap4PageFactory(io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) Param(io.datarouter.web.handler.types.Param) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) SnapshotIdReader(io.datarouter.filesystem.snapshot.reader.SnapshotIdReader) J2HtmlTable(io.datarouter.web.html.j2html.J2HtmlTable) ArrayList(java.util.ArrayList) SnapshotRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotRecord) SnapshotIdReader(io.datarouter.filesystem.snapshot.reader.SnapshotIdReader) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) Twin(io.datarouter.util.tuple.Twin)

Example 3 with SnapshotGroup

use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.

the class SnapshotMerger method merge.

public void merge() {
    Map<SnapshotKey, SnapshotKeyAndNumRecords> summaryByKey = mergeGroup.keyReadOps(false).scanSnapshotKeysAndRootBlocks(readExec, 10).map(SnapshotKeyAndNumRecords::new).toMap(summary -> summary.key);
    while (summaryByKey.size() > 1) {
        SnapshotGroup outputGroup = summaryByKey.size() <= mergeFactor ? destinationGroup : mergeGroup;
        Scanner.of(summaryByKey.values()).minN(SnapshotKeyAndNumRecords.BY_NUM_RECORDS, mergeFactor).map(summary -> summary.key).flush(keys -> {
            SnapshotWriteResult result = combineSnapshots(keys, outputGroup);
            var newSummary = new SnapshotKeyAndNumRecords(result.toSnapshotKeyAndRoot());
            summaryByKey.put(result.key, newSummary);
        }).forEach(summaryByKey::remove);
    }
}
Also used : Scanner(io.datarouter.scanner.Scanner) Arrays(java.util.Arrays) Logger(org.slf4j.Logger) LoggerFactory(org.slf4j.LoggerFactory) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) Supplier(java.util.function.Supplier) SnapshotKeyAndNumRecords(io.datarouter.filesystem.snapshot.group.dto.SnapshotKeyAndNumRecords) SnapshotWriterConfig(io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig) List(java.util.List) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult) ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Map(java.util.Map) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) ExecutorService(java.util.concurrent.ExecutorService) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult) SnapshotKeyAndNumRecords(io.datarouter.filesystem.snapshot.group.dto.SnapshotKeyAndNumRecords) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup)

Example 4 with SnapshotGroup

use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.

the class SnapshotMerger method combineSnapshots.

private SnapshotWriteResult combineSnapshots(List<SnapshotKey> keys, SnapshotGroup outputGroup) {
    SnapshotWriteResult result = Scanner.of(keys).map(key -> new ScanningSnapshotReader(key, readExec, 10, mergeGroup, scanNumBlocks)).collate(reader -> reader.scanLeafRecords(0), SnapshotLeafRecord.KEY_COMPARATOR).deduplicateConsecutiveBy(leafRecord -> leafRecord.key, Arrays::equals).map(SnapshotLeafRecord::entry).batch(10_000).apply(batches -> outputGroup.writeOps().write(writerConfig, batches, writeExec, shouldStop));
    keys.forEach(key -> mergeGroup.deleteOps().deleteSnapshot(key, writeExec, 10));
    logger.warn("combined {}, {}", keys.size(), keys);
    return result;
}
Also used : Scanner(io.datarouter.scanner.Scanner) Arrays(java.util.Arrays) Logger(org.slf4j.Logger) LoggerFactory(org.slf4j.LoggerFactory) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) Supplier(java.util.function.Supplier) SnapshotKeyAndNumRecords(io.datarouter.filesystem.snapshot.group.dto.SnapshotKeyAndNumRecords) SnapshotWriterConfig(io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig) List(java.util.List) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult) ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) Map(java.util.Map) SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) ExecutorService(java.util.concurrent.ExecutorService) ScanningSnapshotReader(io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader) SnapshotLeafRecord(io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord) SnapshotWriteResult(io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult)

Example 5 with SnapshotGroup

use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.

the class DatarouterSnapshotEntriesHandler method entries.

@Handler
public Mav entries(@Param(P_groupId) String groupId, @Param(P_snapshotId) String snapshotId, @Param(P_offset) OptionalLong optOffset, @Param(P_limit) OptionalLong optLimit) {
    var snapshotKey = new SnapshotKey(groupId, snapshotId);
    SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
    DomContent content;
    if (group.getSnapshotEntryDecoderClass() == null) {
        String message = String.format("%s not defined for groupId=%s", SnapshotRecordStringDecoder.class.getSimpleName(), snapshotKey.groupId);
        content = div(message);
    } else {
        long offset = optOffset.orElse(0L);
        long limit = optLimit.orElse(100L);
        content = buildTable(snapshotKey, offset, limit);
    }
    return pageFactory.startBuilder(request).withTitle("Datarouter Filesystem - Snapshot Entries").withRequires(DatarouterWebRequireJsV2.SORTTABLE).withContent(content).buildMav();
}
Also used : SnapshotKey(io.datarouter.filesystem.snapshot.key.SnapshotKey) DomContent(j2html.tags.DomContent) SnapshotGroup(io.datarouter.filesystem.snapshot.group.SnapshotGroup) BaseHandler(io.datarouter.web.handler.BaseHandler)

Aggregations

SnapshotGroup (io.datarouter.filesystem.snapshot.group.SnapshotGroup)7 SnapshotKey (io.datarouter.filesystem.snapshot.key.SnapshotKey)5 BaseHandler (io.datarouter.web.handler.BaseHandler)5 List (java.util.List)4 SnapshotGroups (io.datarouter.filesystem.snapshot.group.SnapshotGroups)3 ScanningSnapshotReader (io.datarouter.filesystem.snapshot.reader.ScanningSnapshotReader)3 Mav (io.datarouter.web.handler.mav.Mav)3 Param (io.datarouter.web.handler.types.Param)3 Bootstrap4PageFactory (io.datarouter.web.html.j2html.bootstrap4.Bootstrap4PageFactory)3 DatarouterWebRequireJsV2 (io.datarouter.web.requirejs.DatarouterWebRequireJsV2)3 Inject (javax.inject.Inject)3 Logger (org.slf4j.Logger)3 LoggerFactory (org.slf4j.LoggerFactory)3 SnapshotKeyAndNumRecords (io.datarouter.filesystem.snapshot.group.dto.SnapshotKeyAndNumRecords)2 SnapshotWriteResult (io.datarouter.filesystem.snapshot.group.dto.SnapshotWriteResult)2 SnapshotLeafRecord (io.datarouter.filesystem.snapshot.reader.record.SnapshotLeafRecord)2 SnapshotWriterConfig (io.datarouter.filesystem.snapshot.writer.SnapshotWriterConfig)2 Scanner (io.datarouter.scanner.Scanner)2 ReflectionTool (io.datarouter.util.lang.ReflectionTool)2 J2HtmlTable (io.datarouter.web.html.j2html.J2HtmlTable)2