Examples with LoadSummary com.baidu.hugegraph.loader.metrics.LoadSummary used on opensource projects

Example 1 with LoadSummary

use of com.baidu.hugegraph.loader.metrics.LoadSummary in project incubator-hugegraph-toolchain by apache.

the class HugeGraphLoader method loadInputs.

private void loadInputs() {
    Printer.printRealtimeProgress(this.context);
    LoadOptions options = this.context.options();
    LoadSummary summary = this.context.summary();
    summary.initMetrics(this.mapping);
    summary.startTotalTimer();
    try {
        if (!options.failureMode) {
            // Load normal data from user supplied input structs
            this.loadInputs(this.mapping.structs());
        } else {
            // Load failure data from generated input structs
            this.loadInputs(this.mapping.structsForFailure(options));
        }
        // Waiting for async worker threads finish
        this.manager.waitFinished();
    } finally {
        summary.calculateTotalTime(ElemType.VERTEX);
        summary.calculateTotalTime(ElemType.EDGE);
        summary.stopTotalTimer();
    }
    Printer.printFinalProgress(this.context);
}

Example 2 with LoadSummary

use of com.baidu.hugegraph.loader.metrics.LoadSummary in project incubator-hugegraph-toolchain by apache.

the class Printer method printSummary.

public static void printSummary(LoadContext context) {
    LoadSummary summary = context.summary();
    // Just log vertices/edges metrics
    log(DIVIDE_LINE);
    log("detail metrics");
    summary.inputMetricsMap().forEach((id, metrics) -> {
        log(EMPTY_LINE);
        log(String.format("input-struct '%s'", id));
        log("read success", metrics.readSuccess());
        log("read failure", metrics.readFailure());
        metrics.vertexMetrics().forEach((label, labelMetrics) -> {
            log(String.format("vertex '%s'", label));
            log("parse success", labelMetrics.parseSuccess());
            log("parse failure", labelMetrics.parseFailure());
            log("insert success", labelMetrics.insertSuccess());
            log("insert failure", labelMetrics.insertFailure());
        });
        metrics.edgeMetrics().forEach((label, labelMetrics) -> {
            log(String.format("edge '%s'", label));
            log("parse success", labelMetrics.parseSuccess());
            log("parse failure", labelMetrics.parseFailure());
            log("insert success", labelMetrics.insertSuccess());
            log("insert failure", labelMetrics.insertFailure());
        });
    });
    // Print and log total vertices/edges metrics
    printAndLog(DIVIDE_LINE);
    printCountReport(LoadReport.collect(summary));
    printAndLog(DIVIDE_LINE);
    printMeterReport(summary);
}

Example 3 with LoadSummary

use of com.baidu.hugegraph.loader.metrics.LoadSummary in project incubator-hugegraph-toolchain by apache.

the class Printer method printFinalProgress.

public static void printFinalProgress(LoadContext context) {
    LoadOptions options = context.options();
    if (!options.printProgress) {
        return;
    }
    LoadSummary summary = context.summary();
    long vertexLoaded = summary.vertexLoaded();
    long edgeLoaded = summary.edgeLoaded();
    System.out.println(vertexLoaded + SLASH + edgeLoaded);
}

Example 4 with LoadSummary

use of com.baidu.hugegraph.loader.metrics.LoadSummary in project incubator-hugegraph-toolchain by apache.

the class Printer method printProgress.

public static void printProgress(LoadContext context, ElemType type, long frequency, int batchSize) {
    LoadSummary summary = context.summary();
    long vertexLoaded = summary.vertexLoaded();
    long edgeLoaded = summary.edgeLoaded();
    if (context.options().printProgress) {
        Printer.printInBackward(vertexLoaded, edgeLoaded);
    }
    long loadSuccess = type.isVertex() ? vertexLoaded : edgeLoaded;
    if (loadSuccess % frequency < batchSize) {
        LOG.info("{} has been loaded: {}, average load rate: {}/s", type.string(), loadSuccess, summary.loadRate(type));
    }
}

Example 5 with LoadSummary

use of com.baidu.hugegraph.loader.metrics.LoadSummary in project incubator-hugegraph-toolchain by apache.

the class TaskManager method submitInSingle.

private void submitInSingle(InputStruct struct, ElementMapping mapping, List<Record> batch) {
    long start = System.currentTimeMillis();
    try {
        this.singleSemaphore.acquire();
    } catch (InterruptedException e) {
        throw new LoadException("Interrupted while waiting to submit %s " + "batch in single mode", e, mapping.type());
    }
    LoadSummary summary = this.context.summary();
    InsertTask task = new SingleInsertTask(this.context, struct, mapping, batch);
    CompletableFuture.runAsync(task, this.singleService).whenComplete((r, e) -> {
        summary.metrics(struct).minusFlighting(batch.size());
        this.singleSemaphore.release();
        long end = System.currentTimeMillis();
        this.context.summary().addTimeRange(mapping.type(), start, end);
    });
}