Examples with GraphElement com.alibaba.maxgraph.compiler.api.schema.GraphElement used on opensource projects

Example 11 with GraphElement

use of com.alibaba.maxgraph.compiler.api.schema.GraphElement in project GraphScope by alibaba.

the class RemoteProxy method getInEdges.

public Iterator<Edge> getInEdges(Set<Vertex> v, String... label) {
    List<Iterator<StoreApi.GraphEdgeReponse>> iterEdgeList = Lists.newArrayList();
    Pair<GraphSchema, Long> schemaPair = schemaFetcher.getSchemaSnapshotPair();
    GraphSchema schema = schemaPair.getLeft();
    long snapshotId = schemaPair.getRight();
    for (Vertex vertex : v) {
        if (label.length == 0) {
            StoreApi.GetInEdgesRequest.Builder req = StoreApi.GetInEdgesRequest.newBuilder();
            req.setSnapshotId(snapshotId).setDstId(vertex.id.id());
            Iterator<StoreApi.GraphEdgeReponse> edgeResponse = stub.withDeadlineAfter(timeout, TimeUnit.SECONDS).getInEdges(req.build());
            iterEdgeList.add(edgeResponse);
        } else {
            for (String labelVal : label) {
                try {
                    GraphElement element = schema.getElement(labelVal);
                    int labelId = element.getLabelId();
                    StoreApi.GetInEdgesRequest.Builder req = StoreApi.GetInEdgesRequest.newBuilder();
                    req.setSnapshotId(snapshotId).setDstId(vertex.id.id()).setTypeId(labelId);
                    Iterator<StoreApi.GraphEdgeReponse> edgeResponse = stub.withDeadlineAfter(timeout, TimeUnit.SECONDS).getInEdges(req.build());
                    iterEdgeList.add(edgeResponse);
                } catch (Exception ignored) {
                }
            }
        }
    }
    return new IteratorList<>(iterEdgeList, new EdgeResponseFunction(schema, this.graph));
}

Example 12 with GraphElement

use of com.alibaba.maxgraph.compiler.api.schema.GraphElement in project GraphScope by alibaba.

the class CommitDataCommand method run.

public void run() {
    MaxGraphClient client = MaxGraphClient.newBuilder().setHosts(graphEndpoint).build();
    Map<Long, DataLoadTarget> tableToTarget = new HashMap<>();
    for (ColumnMappingInfo columnMappingInfo : columnMappingInfos.values()) {
        long tableId = columnMappingInfo.getTableId();
        int labelId = columnMappingInfo.getLabelId();
        GraphElement graphElement = schema.getElement(labelId);
        String label = graphElement.getLabel();
        DataLoadTarget.Builder builder = DataLoadTarget.newBuilder();
        builder.setLabel(label);
        if (graphElement instanceof GraphEdge) {
            builder.setSrcLabel(schema.getElement(columnMappingInfo.getSrcLabelId()).getLabel());
            builder.setDstLabel(schema.getElement(columnMappingInfo.getDstLabelId()).getLabel());
        }
        tableToTarget.put(tableId, builder.build());
    }
    client.commitDataLoad(tableToTarget);
}

Example 13 with GraphElement

use of com.alibaba.maxgraph.compiler.api.schema.GraphElement in project GraphScope by alibaba.

the class OfflineBuild method main.

public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
    String propertiesFile = args[0];
    Properties properties = new Properties();
    try (InputStream is = new FileInputStream(propertiesFile)) {
        properties.load(is);
    }
    String inputPath = properties.getProperty(INPUT_PATH);
    String outputPath = properties.getProperty(OUTPUT_PATH);
    String columnMappingConfigStr = properties.getProperty(COLUMN_MAPPING_CONFIG);
    String graphEndpoint = properties.getProperty(GRAPH_ENDPOINT);
    MaxGraphClient client = MaxGraphClient.newBuilder().setHosts(graphEndpoint).build();
    ObjectMapper objectMapper = new ObjectMapper();
    Map<String, FileColumnMapping> columnMappingConfig = objectMapper.readValue(columnMappingConfigStr, new TypeReference<Map<String, FileColumnMapping>>() {
    });
    List<DataLoadTarget> targets = new ArrayList<>();
    for (FileColumnMapping fileColumnMapping : columnMappingConfig.values()) {
        targets.add(DataLoadTarget.newBuilder().setLabel(fileColumnMapping.getLabel()).setSrcLabel(fileColumnMapping.getSrcLabel()).setDstLabel(fileColumnMapping.getDstLabel()).build());
    }
    GraphSchema schema = client.prepareDataLoad(targets);
    String schemaJson = GraphSchemaMapper.parseFromSchema(schema).toJsonString();
    int partitionNum = client.getPartitionNum();
    Map<String, ColumnMappingInfo> columnMappingInfos = new HashMap<>();
    columnMappingConfig.forEach((fileName, fileColumnMapping) -> {
        columnMappingInfos.put(fileName, fileColumnMapping.toColumnMappingInfo(schema));
    });
    String ldbcCustomize = properties.getProperty(LDBC_CUSTOMIZE, "true");
    long splitSize = Long.valueOf(properties.getProperty(SPLIT_SIZE, "256")) * 1024 * 1024;
    boolean loadAfterBuild = properties.getProperty(LOAD_AFTER_BUILD, "false").equalsIgnoreCase("true");
    boolean skipHeader = properties.getProperty(SKIP_HEADER, "true").equalsIgnoreCase("true");
    Configuration conf = new Configuration();
    conf.setBoolean("mapreduce.map.speculative", false);
    conf.setBoolean("mapreduce.reduce.speculative", false);
    conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERNODE, splitSize);
    conf.setLong(CombineTextInputFormat.SPLIT_MINSIZE_PERRACK, splitSize);
    conf.setStrings(SCHEMA_JSON, schemaJson);
    String mappings = objectMapper.writeValueAsString(columnMappingInfos);
    conf.setStrings(COLUMN_MAPPINGS, mappings);
    conf.setBoolean(LDBC_CUSTOMIZE, ldbcCustomize.equalsIgnoreCase("true"));
    conf.set(SEPARATOR, properties.getProperty(SEPARATOR, "\\|"));
    conf.setBoolean(SKIP_HEADER, skipHeader);
    Job job = Job.getInstance(conf, "build graph data");
    job.setJarByClass(OfflineBuild.class);
    job.setMapperClass(DataBuildMapper.class);
    job.setPartitionerClass(DataBuildPartitioner.class);
    job.setReducerClass(DataBuildReducer.class);
    job.setNumReduceTasks(partitionNum);
    job.setOutputKeyClass(BytesWritable.class);
    job.setOutputValueClass(BytesWritable.class);
    job.setInputFormatClass(CombineTextInputFormat.class);
    CombineTextInputFormat.setMaxInputSplitSize(job, splitSize);
    LazyOutputFormat.setOutputFormatClass(job, SstOutputFormat.class);
    FileInputFormat.addInputPath(job, new Path(inputPath));
    FileInputFormat.setInputDirRecursive(job, true);
    Path outputDir = new Path(outputPath);
    FileOutputFormat.setOutputPath(job, outputDir);
    if (!job.waitForCompletion(true)) {
        System.exit(1);
    }
    FileSystem fs = outputDir.getFileSystem(job.getConfiguration());
    String dataPath = fs.makeQualified(outputDir).toString();
    Map<String, String> outputMeta = new HashMap<>();
    outputMeta.put("endpoint", graphEndpoint);
    outputMeta.put("schema", schemaJson);
    outputMeta.put("mappings", mappings);
    outputMeta.put("datapath", dataPath);
    FSDataOutputStream os = fs.create(new Path(outputDir, "META"));
    os.writeUTF(objectMapper.writeValueAsString(outputMeta));
    os.flush();
    os.close();
    if (loadAfterBuild) {
        logger.info("start ingesting data");
        client.ingestData(dataPath);
        logger.info("commit bulk load");
        Map<Long, DataLoadTarget> tableToTarget = new HashMap<>();
        for (ColumnMappingInfo columnMappingInfo : columnMappingInfos.values()) {
            long tableId = columnMappingInfo.getTableId();
            int labelId = columnMappingInfo.getLabelId();
            GraphElement graphElement = schema.getElement(labelId);
            String label = graphElement.getLabel();
            DataLoadTarget.Builder builder = DataLoadTarget.newBuilder();
            builder.setLabel(label);
            if (graphElement instanceof GraphEdge) {
                builder.setSrcLabel(schema.getElement(columnMappingInfo.getSrcLabelId()).getLabel());
                builder.setDstLabel(schema.getElement(columnMappingInfo.getDstLabelId()).getLabel());
            }
            tableToTarget.put(tableId, builder.build());
        }
        client.commitDataLoad(tableToTarget);
    }
}

Example 14 with GraphElement

use of com.alibaba.maxgraph.compiler.api.schema.GraphElement in project GraphScope by alibaba.

the class MixedOpProcessor method processEstimateManager.

private Object processEstimateManager(EstimateRequest request, long timeout) throws RetryGremlinException {
    Stopwatch timer = Stopwatch.createStarted();
    CostDataStatistics statistics = CostDataStatistics.getInstance();
    TinkerMaxGraph emptyGraph = new TinkerMaxGraph(null, null, null);
    MaxGraphTraversalSource g = (MaxGraphTraversalSource) emptyGraph.traversal();
    GraphSchema graphSchema = schemaFetcher.getSchemaSnapshotPair().getLeft();
    Map<String, Double> vertexCountList = Maps.newHashMap();
    for (GraphElement vertex : graphSchema.getVertexList()) {
        String queryId = String.valueOf(ThreadLocalRandom.current().nextLong());
        GraphTraversal vertexQuery = g.estimateVCount(vertex.getLabel());
        RemoteRpcProcessor remoteRpcProcessor = new DefaultVertexRpcProcessor();
        MemoryResultProcessor resultProcessor = new MemoryResultProcessor(executeConfig.getBatchQuerySize(), resultIterationBatchSize, queryId);
        processQueryTraversal(vertexQuery.toString(), vertexQuery, timeout, queryId, timer, this.httpRpcConnector, remoteRpcProcessor, resultProcessor);
        double countValue = Double.parseDouble(resultProcessor.getResultList().get(0).toString());
        vertexCountList.put(vertex.getLabel(), countValue);
    }
    Map<String, Double> edgeCountList = Maps.newHashMap();
    for (GraphElement edge : graphSchema.getEdgeList()) {
        GraphTraversal edgeQuery = g.estimateECount(edge.getLabel());
        String queryId = String.valueOf(ThreadLocalRandom.current().nextLong());
        RemoteRpcProcessor remoteRpcProcessor = new DefaultVertexRpcProcessor();
        MemoryResultProcessor resultProcessor = new MemoryResultProcessor(executeConfig.getBatchQuerySize(), resultIterationBatchSize, queryId);
        processQueryTraversal(edgeQuery.toString(), edgeQuery, timeout, queryId, timer, this.httpRpcConnector, remoteRpcProcessor, resultProcessor);
        double countValue = Double.parseDouble(resultProcessor.getResultList().get(0).toString());
        edgeCountList.put(edge.getLabel(), countValue);
    }
    for (Map.Entry<String, Double> entry : vertexCountList.entrySet()) {
        statistics.addVertexCount(entry.getKey(), entry.getValue());
    }
    for (Map.Entry<String, Double> entry : edgeCountList.entrySet()) {
        statistics.addEdgeCount(entry.getKey(), entry.getValue());
    }
    return "Estimate vertex/edge count success";
}

Example 15 with GraphElement

use of com.alibaba.maxgraph.compiler.api.schema.GraphElement in project GraphScope by alibaba.

the class EdgeResponseFunction method apply.

@Override
public Edge apply(StoreApi.GraphEdgeReponse edgeReponse) {
    CompositeId eid = new CompositeId(edgeReponse.getEdgeId(), edgeReponse.getTypeId());
    GraphElement element = schema.getElement(eid.typeId());
    String label = element.getLabel();
    Map<String, Object> properties = RpcProcessorUtils.deserializeProperty(edgeReponse.getPros().toByteArray(), element, schema);
    GremlinQuery.VertexId srcId = edgeReponse.getSrcId();
    GremlinQuery.VertexId dstId = edgeReponse.getDstId();
    Iterator<Vertex> vertexIterator = graph.getVertex(Sets.newHashSet(new CompositeId(srcId.getId(), srcId.getTypeId()), new CompositeId(dstId.getId(), dstId.getTypeId())));
    Vertex srcVertex = null, dstVertex = null;
    while (vertexIterator.hasNext()) {
        Vertex vertex = vertexIterator.next();
        if (vertex.id.id() == srcId.getId()) {
            srcVertex = vertex;
        }
        if (vertex.id.id() == dstId.getId()) {
            dstVertex = vertex;
        }
    }
    if (null == srcVertex) {
        try {
            GraphElement graphElement = schema.getElement(srcId.getTypeId());
            srcVertex = new Vertex(new CompositeId(srcId.getId(), srcId.getTypeId()), graphElement.getLabel(), Maps.newHashMap(), graph);
        } catch (Exception ignored) {
            srcVertex = new Vertex(new CompositeId(srcId.getId(), srcId.getTypeId()), "", Maps.newHashMap(), graph);
        }
    }
    if (null == dstVertex) {
        try {
            GraphElement graphElement = schema.getElement(dstId.getTypeId());
            dstVertex = new Vertex(new CompositeId(dstId.getId(), dstId.getTypeId()), graphElement.getLabel(), Maps.newHashMap(), graph);
        } catch (Exception ignored) {
            dstVertex = new Vertex(new CompositeId(dstId.getId(), dstId.getTypeId()), "", Maps.newHashMap(), graph);
        }
    }
    return new Edge(eid, label, properties, srcVertex, dstVertex, this.graph);
}