Examples with LoadOptions com.baidu.hugegraph.loader.executor.LoadOptions used on opensource projects

Example 6 with LoadOptions

use of com.baidu.hugegraph.loader.executor.LoadOptions in project incubator-hugegraph-toolchain by apache.

the class HugeGraphLoader method handleReadFailure.

private void handleReadFailure(InputStruct struct, ReadException e) {
    LOG.error("Read {} error", struct, e);
    this.context.occuredError();
    LoadOptions options = this.context.options();
    if (options.testMode) {
        throw e;
    }
    // Write to current mapping's read failure log
    this.context.failureLogger(struct).write(e);
    long failures = this.context.summary().totalReadFailures();
    if (options.maxReadErrors != Constants.NO_LIMIT && failures >= options.maxReadErrors) {
        Printer.printError("More than %s read error, stop reading and " + "waiting all parse/insert tasks stopped", options.maxReadErrors);
        this.context.stopLoading();
    }
}

Example 7 with LoadOptions

use of com.baidu.hugegraph.loader.executor.LoadOptions in project incubator-hugegraph-toolchain by apache.

the class HugeGraphLoader method loadInputs.

private void loadInputs() {
    Printer.printRealtimeProgress(this.context);
    LoadOptions options = this.context.options();
    LoadSummary summary = this.context.summary();
    summary.initMetrics(this.mapping);
    summary.startTotalTimer();
    try {
        if (!options.failureMode) {
            // Load normal data from user supplied input structs
            this.loadInputs(this.mapping.structs());
        } else {
            // Load failure data from generated input structs
            this.loadInputs(this.mapping.structsForFailure(options));
        }
        // Waiting for async worker threads finish
        this.manager.waitFinished();
    } finally {
        summary.calculateTotalTime(ElemType.VERTEX);
        summary.calculateTotalTime(ElemType.EDGE);
        summary.stopTotalTimer();
    }
    Printer.printFinalProgress(this.context);
}

Example 8 with LoadOptions

use of com.baidu.hugegraph.loader.executor.LoadOptions in project incubator-hugegraph-toolchain by apache.

the class HugeGraphLoader method clearAllDataIfNeeded.

private void clearAllDataIfNeeded() {
    LoadOptions options = this.context.options();
    if (!options.clearAllData) {
        return;
    }
    int requestTimeout = options.timeout;
    options.timeout = options.clearTimeout;
    HugeClient client = HugeClientHolder.create(options);
    String message = "I'm sure to delete all data";
    LOG.info("Prepare to clear the data of graph '{}'", options.graph);
    client.graphs().clearGraph(options.graph, message);
    LOG.info("The graph '{}' has been cleared successfully", options.graph);
    options.timeout = requestTimeout;
    client.close();
}

Example 9 with LoadOptions

use of com.baidu.hugegraph.loader.executor.LoadOptions in project incubator-hugegraph-toolchain by apache.

the class HugeGraphLoader method handleParseFailure.

private void handleParseFailure() {
    LoadOptions options = this.context.options();
    long failures = this.context.summary().totalParseFailures();
    if (options.maxParseErrors != Constants.NO_LIMIT && failures >= options.maxParseErrors) {
        if (this.context.stopped()) {
            return;
        }
        synchronized (this.context) {
            if (!this.context.stopped()) {
                Printer.printError("More than %s parse error, stop " + "parsing and waiting all insert tasks " + "stopped", options.maxParseErrors);
                this.context.stopLoading();
            }
        }
    }
}

Example 10 with LoadOptions

use of com.baidu.hugegraph.loader.executor.LoadOptions in project incubator-hugegraph-toolchain by apache.

the class FileLoadTest method testClearSchemaBeforeLoad.

@Test
public void testClearSchemaBeforeLoad() {
    LoadOptions options = new LoadOptions();
    options.host = Constants.HTTP_PREFIX + SERVER;
    options.port = PORT;
    options.graph = GRAPH;
    HugeClient client = HugeClientHolder.create(options);
    SchemaManager schema = client.schema();
    schema.propertyKey("name").asText().ifNotExist().create();
    schema.propertyKey("age").asInt().ifNotExist().create();
    // The old datatype of city is int
    schema.propertyKey("city").asInt().ifNotExist().create();
    schema.vertexLabel("person").properties("name", "age", "city").primaryKeys("name").ifNotExist().create();
    // Actual city datatype is String
    ioUtil.write("vertex_person.csv", "name,age,city", "marko,29,Beijing", "vadas,27,Hongkong", "josh,32,Beijing", "peter,35,Shanghai", "\"li,nary\",26,\"Wu,han\"");
    String[] args1 = new String[] { "-f", structPath("clear_schema_before_load/struct.json"), "-g", GRAPH, "-h", SERVER, "--batch-insert-threads", "2", "--test-mode", "true" };
    Assert.assertThrows(ParseException.class, () -> {
        HugeGraphLoader.main(args1);
    }, (e) -> {
        String msg = e.getMessage();
        Assert.assertTrue(msg.startsWith("Failed to convert value"));
        Assert.assertTrue(msg.endsWith("to Number"));
    });
    String[] args2 = new String[] { "-f", structPath("clear_schema_before_load/struct.json"), "-s", configPath("clear_schema_before_load/schema.groovy"), "-g", GRAPH, "-h", SERVER, "--clear-all-data", "true", "--batch-insert-threads", "2", "--test-mode", "true" };
    HugeGraphLoader.main(args2);
    List<Vertex> vertices = CLIENT.graph().listVertices();
    Assert.assertEquals(5, vertices.size());
    client.close();
}