use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.
the class DatarouterSnapshotEntriesHandler method buildTable.
private DomContent buildTable(SnapshotKey snapshotKey, long offset, long limit) {
SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
var reader = new ScanningSnapshotReader(snapshotKey, exec, 2, groups, 1);
SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
List<SnapshotRecordStrings> rows = reader.scan(0).skip(offset).limit(limit).map(decoder::decode).list();
var table = new J2HtmlTable<SnapshotRecordStrings>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("id", row -> row.id).withColumn(decoder.keyName(), row -> row.key).withColumn(decoder.valueName(), row -> {
if (row.value == null) {
return "";
} else if (row.value.length() < 64) {
return row.value;
} else {
return row.value.subSequence(0, 64) + "...";
}
}).withHtmlColumn("details", row -> {
String href = new URIBuilder().setPath(request.getContextPath() + snapshotPaths.datarouter.snapshot.individual.entry.toSlashedString()).addParameter(DatarouterSnapshotEntryHandler.P_groupId, snapshotKey.groupId).addParameter(DatarouterSnapshotEntryHandler.P_snapshotId, snapshotKey.snapshotId).addParameter(DatarouterSnapshotEntryHandler.P_id, Long.toString(row.id)).toString();
return td(a("view").withHref(href));
}).build(rows);
return table;
}
use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.
the class DatarouterSnapshotEntryHandler method buildTable.
private ContainerTag<?> buildTable(SnapshotKey snapshotKey, long id) {
SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
var reader = new SnapshotIdReader(snapshotKey, groups);
SnapshotRecord record = reader.getRecord(id);
SnapshotRecordStringDecoder decoder = ReflectionTool.create(group.getSnapshotEntryDecoderClass());
SnapshotRecordStrings decoded = decoder.decode(record);
List<Twin<String>> rows = new ArrayList<>();
rows.add(new Twin<>("id", Long.toString(record.id)));
rows.add(new Twin<>(decoder.keyName(), decoded.key));
rows.add(new Twin<>(decoder.valueName(), decoded.value));
IntStream.range(0, decoded.columnValues.size()).mapToObj(column -> new Twin<>(decoder.columnValueName(column), decoded.columnValues.get(column))).forEach(rows::add);
var table = new J2HtmlTable<Twin<String>>().withClasses("sortable table table-sm table-striped my-4 border").withColumn("field", twin -> twin.getLeft()).withColumn("value", twin -> twin.getRight()).build(rows);
return table;
}
use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.
the class SnapshotMerger method merge.
public void merge() {
Map<SnapshotKey, SnapshotKeyAndNumRecords> summaryByKey = mergeGroup.keyReadOps(false).scanSnapshotKeysAndRootBlocks(readExec, 10).map(SnapshotKeyAndNumRecords::new).toMap(summary -> summary.key);
while (summaryByKey.size() > 1) {
SnapshotGroup outputGroup = summaryByKey.size() <= mergeFactor ? destinationGroup : mergeGroup;
Scanner.of(summaryByKey.values()).minN(SnapshotKeyAndNumRecords.BY_NUM_RECORDS, mergeFactor).map(summary -> summary.key).flush(keys -> {
SnapshotWriteResult result = combineSnapshots(keys, outputGroup);
var newSummary = new SnapshotKeyAndNumRecords(result.toSnapshotKeyAndRoot());
summaryByKey.put(result.key, newSummary);
}).forEach(summaryByKey::remove);
}
}
use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.
the class SnapshotMerger method combineSnapshots.
private SnapshotWriteResult combineSnapshots(List<SnapshotKey> keys, SnapshotGroup outputGroup) {
SnapshotWriteResult result = Scanner.of(keys).map(key -> new ScanningSnapshotReader(key, readExec, 10, mergeGroup, scanNumBlocks)).collate(reader -> reader.scanLeafRecords(0), SnapshotLeafRecord.KEY_COMPARATOR).deduplicateConsecutiveBy(leafRecord -> leafRecord.key, Arrays::equals).map(SnapshotLeafRecord::entry).batch(10_000).apply(batches -> outputGroup.writeOps().write(writerConfig, batches, writeExec, shouldStop));
keys.forEach(key -> mergeGroup.deleteOps().deleteSnapshot(key, writeExec, 10));
logger.warn("combined {}, {}", keys.size(), keys);
return result;
}
use of io.datarouter.filesystem.snapshot.group.SnapshotGroup in project datarouter by hotpads.
the class DatarouterSnapshotEntriesHandler method entries.
@Handler
public Mav entries(@Param(P_groupId) String groupId, @Param(P_snapshotId) String snapshotId, @Param(P_offset) OptionalLong optOffset, @Param(P_limit) OptionalLong optLimit) {
var snapshotKey = new SnapshotKey(groupId, snapshotId);
SnapshotGroup group = groups.getGroup(snapshotKey.groupId);
DomContent content;
if (group.getSnapshotEntryDecoderClass() == null) {
String message = String.format("%s not defined for groupId=%s", SnapshotRecordStringDecoder.class.getSimpleName(), snapshotKey.groupId);
content = div(message);
} else {
long offset = optOffset.orElse(0L);
long limit = optLimit.orElse(100L);
content = buildTable(snapshotKey, offset, limit);
}
return pageFactory.startBuilder(request).withTitle("Datarouter Filesystem - Snapshot Entries").withRequires(DatarouterWebRequireJsV2.SORTTABLE).withContent(content).buildMav();
}
Aggregations