Search in sources :

Example 6 with GraphSchema

use of com.alibaba.maxgraph.compiler.api.schema.GraphSchema in project GraphScope by alibaba.

the class MaxGraphImpl method addEdge.

@Override
public Edge addEdge(String label, Vertex src, Vertex dst, Map<String, Object> properties) {
    GraphSchema schema = getSchema();
    int edgeLabelId = schema.getElement(label).getLabelId();
    EdgeKind edgeKind = EdgeKind.newBuilder().setEdgeLabelId(new LabelId(edgeLabelId)).setSrcVertexLabelId(new LabelId(src.id.typeId())).setDstVertexLabelId(new LabelId(dst.id.typeId())).build();
    long innerId = ++startEdgeInnerId;
    EdgeId edgeId = new EdgeId(new VertexId(src.id.id()), new VertexId(dst.id.id()), innerId);
    EdgeTarget edgeTarget = new EdgeTarget(edgeKind, edgeId);
    DataRecord dataRecord = new DataRecord(edgeTarget, properties);
    WriteRequest writeRequest = new WriteRequest(OperationType.OVERWRITE_EDGE, dataRecord);
    graphWriter.writeBatch(getClass().getCanonicalName(), this.writeSession, Arrays.asList(writeRequest));
    return null;
}
Also used : EdgeKind(com.alibaba.maxgraph.sdkcommon.schema.EdgeKind) EdgeId(com.alibaba.graphscope.groot.operation.EdgeId) VertexId(com.alibaba.graphscope.groot.operation.VertexId) LabelId(com.alibaba.maxgraph.sdkcommon.schema.LabelId) GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema)

Example 7 with GraphSchema

use of com.alibaba.maxgraph.compiler.api.schema.GraphSchema in project GraphScope by alibaba.

the class RemoteProxy method scanEdge.

public Iterator<Edge> scanEdge(Set<String> labelList) {
    Pair<GraphSchema, Long> pair = schemaFetcher.getSchemaSnapshotPair();
    Set<Integer> labelIdList = Sets.newHashSet();
    if (null == labelList || labelList.isEmpty()) {
        labelIdList.add(0);
    } else {
        for (String label : labelList) {
            try {
                labelIdList.add(pair.getLeft().getElement(label).getLabelId());
            } catch (Exception ignored) {
            }
        }
    }
    if (labelIdList.isEmpty()) {
        return new ArrayList<Edge>().iterator();
    }
    List<Iterator<StoreApi.GraphEdgeReponse>> resList = Lists.newArrayList();
    for (int labelId : labelIdList) {
        StoreApi.ScanEdgeRequest.Builder req = StoreApi.ScanEdgeRequest.newBuilder();
        req.setSnapshotId(pair.getRight()).setOffset(0).setLimit(Integer.MAX_VALUE).setTypeId(labelId);
        resList.add(stub.withDeadlineAfter(timeout, TimeUnit.SECONDS).scanEdges(req.build()));
    }
    return new IteratorList<>(resList, new EdgeResponseFunction(pair.getLeft(), this.graph));
}
Also used : EdgeResponseFunction(com.alibaba.maxgraph.iterator.function.EdgeResponseFunction) StoreApi(com.alibaba.maxgraph.proto.StoreApi) GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema) IteratorList(com.alibaba.maxgraph.iterator.IteratorList) Edge(com.alibaba.maxgraph.structure.Edge)

Example 8 with GraphSchema

use of com.alibaba.maxgraph.compiler.api.schema.GraphSchema in project GraphScope by alibaba.

the class GraphWriter method writeBatch.

public void writeBatch(String requestId, String writeSession, List<WriteRequest> writeRequests, CompletionCallback<Long> callback) {
    this.pendingWriteCount.incrementAndGet();
    GraphSchema schema = snapshotCache.getSnapshotWithSchema().getGraphDef();
    OperationBatch.Builder batchBuilder = OperationBatch.newBuilder();
    for (WriteRequest writeRequest : writeRequests) {
        OperationType operationType = writeRequest.getOperationType();
        DataRecord dataRecord = writeRequest.getDataRecord();
        switch(operationType) {
            case OVERWRITE_VERTEX:
                addOverwriteVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case UPDATE_VERTEX:
                addUpdateVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case DELETE_VERTEX:
                addDeleteVertexOperation(batchBuilder, schema, dataRecord);
                break;
            case OVERWRITE_EDGE:
                addOverwriteEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            case UPDATE_EDGE:
                addUpdateEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            case DELETE_EDGE:
                addDeleteEdgeOperation(batchBuilder, schema, dataRecord);
                break;
            default:
                throw new IllegalArgumentException("Invalid operationType [" + operationType + "]");
        }
    }
    OperationBatch operationBatch = batchBuilder.build();
    int writeQueueId = getWriteQueueId(writeSession);
    int ingestorId = this.metaService.getIngestorIdForQueue(writeQueueId);
    long startTimeNano = System.nanoTime();
    this.ingestWriteClients.getClient(ingestorId).writeIngestorAsync(requestId, writeQueueId, operationBatch, new CompletionCallback<Long>() {

        @Override
        public void onCompleted(Long res) {
            long writeSnapshotId = res;
            lastWrittenSnapshotId.updateAndGet(x -> x < writeSnapshotId ? writeSnapshotId : x);
            writeRequestsTotal.addAndGet(writeRequests.size());
            finish();
            callback.onCompleted(res);
        }

        @Override
        public void onError(Throwable t) {
            finish();
            callback.onError(t);
        }

        void finish() {
            long ingestorCompleteTimeNano = System.nanoTime();
            ingestorBlockTimeNano.addAndGet(ingestorCompleteTimeNano - startTimeNano);
            pendingWriteCount.decrementAndGet();
        }
    });
}
Also used : VertexRecordKey(com.alibaba.maxgraph.sdkcommon.common.VertexRecordKey) GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema) PkHashUtils(com.alibaba.maxgraph.sdkcommon.util.PkHashUtils) EdgeRecordKey(com.alibaba.maxgraph.sdkcommon.common.EdgeRecordKey) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) DataType(com.alibaba.maxgraph.compiler.api.schema.DataType) ArrayList(java.util.ArrayList) WriteSessionUtil(com.alibaba.maxgraph.common.util.WriteSessionUtil) PropertyDefNotFoundException(com.alibaba.maxgraph.compiler.api.exception.PropertyDefNotFoundException) MetricsCollector(com.alibaba.graphscope.groot.metrics.MetricsCollector) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricsAgent(com.alibaba.graphscope.groot.metrics.MetricsAgent) LabelId(com.alibaba.maxgraph.sdkcommon.schema.LabelId) VertexId(com.alibaba.graphscope.groot.operation.VertexId) com.alibaba.graphscope.groot.operation.dml(com.alibaba.graphscope.groot.operation.dml) Map(java.util.Map) PropertyValue(com.alibaba.maxgraph.sdkcommon.schema.PropertyValue) MaxGraphException(com.alibaba.maxgraph.compiler.api.exception.MaxGraphException) EdgeId(com.alibaba.graphscope.groot.operation.EdgeId) CompletionCallback(com.alibaba.graphscope.groot.CompletionCallback) GraphProperty(com.alibaba.maxgraph.compiler.api.schema.GraphProperty) MetaService(com.alibaba.graphscope.groot.meta.MetaService) EdgeKind(com.alibaba.maxgraph.sdkcommon.schema.EdgeKind) OperationType(com.alibaba.graphscope.groot.operation.OperationType) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) AtomicLong(java.util.concurrent.atomic.AtomicLong) RoleClients(com.alibaba.graphscope.groot.rpc.RoleClients) List(java.util.List) SnapshotCache(com.alibaba.graphscope.groot.SnapshotCache) IngestorWriteClient(com.alibaba.graphscope.groot.frontend.IngestorWriteClient) GraphElement(com.alibaba.maxgraph.compiler.api.schema.GraphElement) GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema) OperationBatch(com.alibaba.graphscope.groot.operation.OperationBatch) AtomicLong(java.util.concurrent.atomic.AtomicLong) OperationType(com.alibaba.graphscope.groot.operation.OperationType)

Example 9 with GraphSchema

use of com.alibaba.maxgraph.compiler.api.schema.GraphSchema in project GraphScope by alibaba.

the class RemoteTestGraph method dropData.

private void dropData() {
    GraphSchema schema = sdkClient.dropSchema();
    logger.info("drop schema: " + ((GraphDef) schema).toProto().toString());
}
Also used : GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema)

Example 10 with GraphSchema

use of com.alibaba.maxgraph.compiler.api.schema.GraphSchema in project GraphScope by alibaba.

the class MixedTraversalOpProcessor method doProcessTraversal.

private Long doProcessTraversal(Context context, Object object, Graph graph, long timeout, String queryId, Stopwatch timer) {
    Pair<GraphSchema, Long> snapshotSchema;
    GraphSchema schema;
    Long resultNum = 0L;
    if (object instanceof GraphTraversal.Admin || object instanceof DfsTraversal) {
        QueryStatus queryStatus;
        GraphTraversal.Admin traversal = (object instanceof GraphTraversal.Admin) ? GraphTraversal.Admin.class.cast(object) : (DfsTraversal.class.cast(object)).getTraversal();
        String queryString = traversal.toString();
        logger.info("Receive traversal query=>" + queryString);
        if (!traversal.isLocked()) {
            traversal.getStrategies().removeStrategies(ProfileStrategy.class, MxGraphStepStrategy.class, FilterRankingStrategy.class);
        }
        traversal.applyStrategies();
        NettyVertexRpcProcessor nettyVertexRpcProcessor;
        QueryFlowManager queryFlowManager;
        // 保证一查看到snapshotId就开始维护query_status
        synchronized (queryCallbackManager) {
            snapshotSchema = schemaFetcher.getSchemaSnapshotPair();
            queryStatus = queryCallbackManager.beforeExecution(snapshotSchema.getRight());
        }
        schema = snapshotSchema.getLeft();
        LogicalPlanOptimizer planOptimizer = new LogicalPlanOptimizer(new OptimizeConfig(), this.globalPullGraphFlag, schema, snapshotSchema.getRight(), this.lambdaEnableFlag);
        final int resultIterationBatchSize = (Integer) context.getRequestMessage().optionalArgs(Tokens.ARGS_BATCH_SIZE).orElse(this.resultIterationBatchSize);
        nettyVertexRpcProcessor = new NettyTraverserVertexProcessor(context, resultIterationBatchSize, false);
        try {
            queryFlowManager = (object instanceof GraphTraversal.Admin) ? planOptimizer.build(GraphTraversal.class.cast(traversal)) : planOptimizer.build(DfsTraversal.class.cast(object));
        } catch (IllegalArgumentException iae) {
            if (iae.getMessage().contains("MaxGraphIoStep")) {
                logger.info("do maxgraph io step");
                while (traversal.hasNext()) {
                    logger.info("maxgraph io hasNext");
                }
                nettyVertexRpcProcessor.finish(ResponseStatusCode.SUCCESS);
                return 0L;
            }
            throw iae;
        }
        try {
            boolean isLambdaExisted = TraversalHelper.anyStepRecursively(s -> s instanceof LambdaHolder, (Traversal.Admin<?, ?>) traversal);
            queryFlowManager.getQueryFlow().setFrontId(serverId);
            if (this.lambdaEnableFlag && isLambdaExisted) {
                final ObjectMapper mapper = GraphSONMapper.build().version(GraphSONVersion.V3_0).addCustomModule(GraphSONXModuleV3d0.build().create(false)).create().createMapper();
                Bytecode bytecode = (Bytecode) context.getRequestMessage().getArgs().get(Tokens.ARGS_GREMLIN);
                byte[] bytecodeByte = mapper.writeValueAsBytes(bytecode);
                queryFlowManager.getQueryFlow().setLambdaExisted(isLambdaExisted).setBytecode(ByteString.copyFrom(bytecodeByte));
            }
            GremlinResultTransform gremlinResultTransform = new GremlinResultTransform(remoteRpcConnector, nettyVertexRpcProcessor, this.graph, queryFlowManager.getResultValueType(), vertexCacheFlag);
            NettyResultProcessor nettyResultProcessor = new NettyResultProcessor(queryId, traversal.toString(), context, new ExecuteConfig().getBatchQuerySize(), resultIterationBatchSize, false);
            nettyResultProcessor.setSchema(schema);
            nettyResultProcessor.setResultTransform(gremlinResultTransform);
            nettyResultProcessor.setLabelIndexNameList(queryFlowManager.getTreeNodeLabelManager().getUserIndexLabelList());
            TimelyQuery timelyQuery = new TimelyQuery(queryFlowManager, nettyResultProcessor, this.graph);
            Logging.query(this.graphName, FRONTEND, this.serverId, queryId, QueryType.EXECUTE, QueryEvent.PLAN_GENERATED, timer.elapsed(TimeUnit.NANOSECONDS), null, null, "");
            timelyExecutor.execute(timelyQuery, schema, timeout, queryId);
            resultNum = nettyResultProcessor.total();
        } catch (JsonProcessingException e) {
            e.printStackTrace();
        } finally {
            queryCallbackManager.afterExecution(queryStatus);
        }
    } else {
        throw new IllegalArgumentException("Not support to process=>" + object);
    }
    return resultNum;
}
Also used : TimelyQuery(com.alibaba.maxgraph.compiler.query.TimelyQuery) DfsTraversal(com.alibaba.maxgraph.compiler.dfs.DfsTraversal) LambdaHolder(org.apache.tinkerpop.gremlin.process.traversal.step.LambdaHolder) GraphTraversal(org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal) Traversal(org.apache.tinkerpop.gremlin.process.traversal.Traversal) DfsTraversal(com.alibaba.maxgraph.compiler.dfs.DfsTraversal) QueryFlowManager(com.alibaba.maxgraph.compiler.optimizer.QueryFlowManager) ByteString(com.google.protobuf.ByteString) QueryStatus(com.alibaba.maxgraph.api.query.QueryStatus) GraphSchema(com.alibaba.maxgraph.compiler.api.schema.GraphSchema) GraphTraversal(org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal) Bytecode(org.apache.tinkerpop.gremlin.process.traversal.Bytecode) JsonProcessingException(org.apache.tinkerpop.shaded.jackson.core.JsonProcessingException) ObjectMapper(org.apache.tinkerpop.shaded.jackson.databind.ObjectMapper) OptimizeConfig(com.alibaba.maxgraph.compiler.optimizer.OptimizeConfig) LogicalPlanOptimizer(com.alibaba.maxgraph.compiler.optimizer.LogicalPlanOptimizer) ExecuteConfig(com.alibaba.maxgraph.compiler.executor.ExecuteConfig)

Aggregations

GraphSchema (com.alibaba.maxgraph.compiler.api.schema.GraphSchema)26 GraphElement (com.alibaba.maxgraph.compiler.api.schema.GraphElement)12 List (java.util.List)9 Map (java.util.Map)8 Lists (com.google.common.collect.Lists)7 Sets (com.google.common.collect.Sets)7 Set (java.util.Set)7 TreeNode (com.alibaba.maxgraph.compiler.tree.TreeNode)6 Maps (com.google.common.collect.Maps)6 Collectors (java.util.stream.Collectors)6 EdgeRelation (com.alibaba.maxgraph.compiler.api.schema.EdgeRelation)5 GraphEdge (com.alibaba.maxgraph.compiler.api.schema.GraphEdge)5 EdgeOtherVertexTreeNode (com.alibaba.maxgraph.compiler.tree.EdgeOtherVertexTreeNode)5 EdgeTreeNode (com.alibaba.maxgraph.compiler.tree.EdgeTreeNode)5 EdgeVertexTreeNode (com.alibaba.maxgraph.compiler.tree.EdgeVertexTreeNode)5 VertexTreeNode (com.alibaba.maxgraph.compiler.tree.VertexTreeNode)5 SourceTreeNode (com.alibaba.maxgraph.compiler.tree.source.SourceTreeNode)5 Direction (org.apache.tinkerpop.gremlin.structure.Direction)5 JSONObject (com.alibaba.fastjson.JSONObject)4 SchemaFetcher (com.alibaba.maxgraph.compiler.api.schema.SchemaFetcher)4