Examples with GraphRecordStore au.gov.asd.tac.constellation.graph.processing.GraphRecordStore used on opensource projects

Example 16 with GraphRecordStore

use of au.gov.asd.tac.constellation.graph.processing.GraphRecordStore in project constellation by constellation-app.

the class ContractedCompositeNodeState method expand.

/**
 * Expand the composite represented by this state, using the supplied graph
 * write lock.
 *
 * @param wg The graph write lock with which to perform the expansion.
 * @param vxId The Graph ID of the node that is being expanded. The caller
 * must ensure that this is the state object corresponding to this node!
 * @return A list of graph IDs of the constituent nodes of the now expanded
 * composite.
 */
public List<Integer> expand(final GraphWriteMethods wg, final int vxId) {
    // Create rows in the expansion record store for transactions that are currently connected to the composite node and should be connected to expnaded nodes.
    final int uniqueIdAttr = VisualConcept.TransactionAttribute.IDENTIFIER.get(wg);
    final int compositeStateAttr = AnalyticConcept.VertexAttribute.COMPOSITE_STATE.get(wg);
    final int selectedAttr = VisualConcept.VertexAttribute.SELECTED.get(wg);
    final int xAttr = VisualConcept.VertexAttribute.X.get(wg);
    final int yAttr = VisualConcept.VertexAttribute.Y.get(wg);
    final int zAttr = VisualConcept.VertexAttribute.Z.get(wg);
    // Clear the composite state we are currently expanding!
    wg.setObjectValue(compositeStateAttr, vxId, null);
    // Create the cotnraction store and get the stored id of the contracted composite ndoe
    final String[] contractedId = new String[1];
    final RecordStore contractionStore = GraphRecordStoreUtilities.copySpecifiedVertex(wg, null, vxId, contractedId);
    final RecordStore addToGraphStore = new GraphRecordStore();
    addToGraphStore.add(constituentNodeStore);
    // Copy the relevant transactions from the composite to the constituent node record store
    GraphRecordStoreUtilities.copyTransactionsFromComposite(wg, addToGraphStore, vxId, contractedId[0], affectedExpandedIds, new GraphAttribute(wg, uniqueIdAttr));
    // Add the now complete expanded composite node store to the graph (do not initialise or complete with schema)
    final Map<String, Integer> vertexMap = new HashMap<>();
    final List<Integer> expandedVerts = GraphRecordStoreUtilities.addRecordStoreToGraph(wg, addToGraphStore, false, false, null, vertexMap, null);
    // Create the expanded composite state for each expanded node and add it to those nodes.
    vertexMap.entrySet().forEach(entry -> {
        final int id = entry.getValue();
        final ExpandedCompositeNodeState expansionState = new ExpandedCompositeNodeState(contractionStore, contractedId[0], affectedExpandedIds.contains(entry.getKey()), expandedVerts.size());
        wg.setObjectValue(compositeStateAttr, id, new CompositeNodeState(id, expansionState));
    });
    // Correct the x,y,z coordinates of the nodes in the zonsitutentNodeStore in case the composite node has been moved.
    final float x = wg.getFloatValue(xAttr, vxId);
    final float y = wg.getFloatValue(yAttr, vxId);
    final float z = wg.getFloatValue(zAttr, vxId);
    final boolean selected = wg.getBooleanValue(selectedAttr, vxId);
    vertexMap.values().forEach(id -> {
        final float currentX = wg.getFloatValue(xAttr, id);
        final float currentY = wg.getFloatValue(yAttr, id);
        final float currentZ = wg.getFloatValue(zAttr, id);
        wg.setFloatValue(xAttr, id, currentX - mean[0] + x);
        wg.setFloatValue(yAttr, id, currentY - mean[1] + y);
        wg.setFloatValue(zAttr, id, currentZ - mean[2] + z);
        wg.setBooleanValue(selectedAttr, id, selected);
    });
    // Delete the vertex represented by this composite node state
    wg.removeVertex(vxId);
    return new ArrayList<>(vertexMap.values());
}

Example 17 with GraphRecordStore

use of au.gov.asd.tac.constellation.graph.processing.GraphRecordStore in project constellation by constellation-app.

the class ExpandedCompositeNodeState method contract.

/**
 * Contract the composite represented by this state, using the supplied
 * graph write lock.
 *
 * @param wg The graph write lock with which to perform the expansion.
 * @return The graph ID of the now contracted composite node.
 */
public int contract(final GraphWriteMethods wg) {
    final int uniqueIdAttr = VisualConcept.TransactionAttribute.IDENTIFIER.get(wg);
    final int compositeStateAttr = AnalyticConcept.VertexAttribute.COMPOSITE_STATE.get(wg);
    final int selectedAttr = VisualConcept.VertexAttribute.SELECTED.get(wg);
    final int xAttr = VisualConcept.VertexAttribute.X.get(wg);
    final int yAttr = VisualConcept.VertexAttribute.Y.get(wg);
    final int zAttr = VisualConcept.VertexAttribute.Z.get(wg);
    final RecordStore constituentNodeStore = new GraphRecordStore();
    final List<String> expandedIds = new ArrayList<>();
    final List<String> affectedExpandedIds = new ArrayList<>();
    // Iterate through each vertex on the graph, and if it has an expanded composite state with the same composite id
    // as this state, perform the single vertex contraction. Keep track of x, y and z as we go so that we can set
    // the coordinates of the contracted composite to be at the centre of the expanded constituents.
    float x = 0;
    float y = 0;
    float z = 0;
    final Map<Integer, String> idToCopiedId = new HashMap<>();
    final int vertexCount = wg.getVertexCount();
    int numConstituents = 0;
    boolean constitutentsSelected = false;
    for (int i = 0; i < vertexCount; i++) {
        final int vxId = wg.getVertex(i);
        final CompositeNodeState cns = ((CompositeNodeState) wg.getObjectValue(compositeStateAttr, vxId));
        if (cns != null && cns.expandedState != null && cns.expandedState.getCompositeId().equals(compositeId)) {
            x += wg.getFloatValue(xAttr, vxId);
            y += wg.getFloatValue(yAttr, vxId);
            z += wg.getFloatValue(zAttr, vxId);
            constitutentsSelected |= wg.getBooleanValue(selectedAttr, vxId);
            // Clear the composite state we are about to expand
            wg.setObjectValue(compositeStateAttr, vxId, null);
            // Expand the composite state
            cns.expandedState.contractSingleVertex(wg, vxId, constituentNodeStore, expandedIds, affectedExpandedIds, idToCopiedId);
            numConstituents++;
        }
    }
    if (numConstituents != 0) {
        x /= numConstituents;
        y /= numConstituents;
        z /= numConstituents;
    }
    // The contracted state of this composite node, to be set on the composite node after it has been created.
    final float[] mean = new float[] { x, y, z };
    final ContractedCompositeNodeState contractionState = new ContractedCompositeNodeState(constituentNodeStore, expandedIds, affectedExpandedIds, mean);
    final RecordStore addToGraphStore = new GraphRecordStore();
    addToGraphStore.add(compositeNodeStore);
    // Add the transactions from the expanded nodes to non-constituent nodes to the contraction record store
    idToCopiedId.entrySet().forEach(entry -> GraphRecordStoreUtilities.copyTransactionsToComposite(wg, addToGraphStore, entry.getKey(), entry.getValue(), idToCopiedId.keySet(), compositeId, new GraphAttribute(wg, uniqueIdAttr)));
    // Remove each expanded vertex from the graph
    idToCopiedId.keySet().forEach(wg::removeVertex);
    // Add all the transactions between the expanded nodes to the expansion record store
    // Add the contraction record store to the graph, which creates the composite node and all its relevant transactions
    int contractedVert = GraphRecordStoreUtilities.addRecordStoreToGraph(wg, addToGraphStore, false, false, null).get(0);
    // Set the x,y,z and composite node state for the newly added composite node.
    wg.setFloatValue(xAttr, contractedVert, x);
    wg.setFloatValue(yAttr, contractedVert, y);
    wg.setFloatValue(zAttr, contractedVert, z);
    wg.setBooleanValue(selectedAttr, contractedVert, constitutentsSelected);
    wg.setObjectValue(compositeStateAttr, contractedVert, new CompositeNodeState(contractedVert, contractionState));
    return contractedVert;
}

Example 18 with GraphRecordStore

use of au.gov.asd.tac.constellation.graph.processing.GraphRecordStore in project constellation by constellation-app.

the class ExtractTypesFromTextPlugin method query.

@Override
protected RecordStore query(final RecordStore query, final PluginInteraction interaction, final PluginParameters parameters) throws InterruptedException, PluginException {
    final RecordStore result = new GraphRecordStore();
    interaction.setProgress(0, 0, "Importing...", true);
    final Map<String, PluginParameter<?>> extractEntityParameters = parameters.getParameters();
    final String text = extractEntityParameters.get(TEXT_PARAMETER_ID).getStringValue();
    if (text == null) {
        throw new PluginException(PluginNotificationLevel.ERROR, "No text provided from which to extract types.");
    }
    final List<ExtractedVertexType> extractedTypes = SchemaVertexTypeUtilities.extractVertexTypes(text);
    final Map<String, SchemaVertexType> identifiers = new HashMap<>();
    extractedTypes.forEach(extractedType -> identifiers.put(extractedType.getIdentifier(), extractedType.getType()));
    for (final String identifier : identifiers.keySet()) {
        result.add();
        result.set(GraphRecordStoreUtilities.SOURCE + VisualConcept.VertexAttribute.IDENTIFIER, identifier);
        result.set(GraphRecordStoreUtilities.SOURCE + AnalyticConcept.VertexAttribute.TYPE, identifiers.get(identifier));
        result.set(GraphRecordStoreUtilities.SOURCE + AnalyticConcept.VertexAttribute.SEED, "true");
    }
    ConstellationLoggerHelper.createPropertyBuilder(this, result.getAll(GraphRecordStoreUtilities.SOURCE + VisualConcept.VertexAttribute.IDENTIFIER), ConstellationLoggerHelper.SUCCESS);
    interaction.setProgress(1, 0, "Completed successfully - imported " + result.size() + " entities.", true);
    return result;
}

Example 19 with GraphRecordStore

use of au.gov.asd.tac.constellation.graph.processing.GraphRecordStore in project constellation by constellation-app.

the class TSVDropper method drop.

@Override
public BiConsumer<Graph, DropInfo> drop(final DropTargetDropEvent dtde) {
    // Only work on files
    final Transferable transferable = dtde.getTransferable();
    if (transferable.isDataFlavorSupported(DataFlavor.javaFileListFlavor)) {
        try {
            // Get the data as a list of files
            final Object data = dtde.getTransferable().getTransferData(DataFlavor.javaFileListFlavor);
            // data will be list of files which extends Object type
            @SuppressWarnings("unchecked") final List<File> files = (List<File>) data;
            // Create a record store to hold the combined results
            final RecordStore recordStore = new GraphRecordStore();
            boolean badData = false;
            // Process each file...
            for (final File file : files) {
                // Only process files
                if (file.isFile()) {
                    // Only process files that have a .tsv or .tsv.gz extension
                    // If any file does not have this extension then reject all the files.
                    final InputStream in;
                    if (StringUtils.endsWithIgnoreCase(file.getName(), FileExtensionConstants.TAB_SEPARATED_VALUE + FileExtensionConstants.GZIP)) {
                        in = new GZIPInputStream(new FileInputStream(file));
                    } else if (StringUtils.endsWithIgnoreCase(file.getName(), FileExtensionConstants.TAB_SEPARATED_VALUE)) {
                        in = new FileInputStream(file);
                    } else {
                        badData = true;
                        break;
                    }
                    // Open a reader so that we can read the file line by line
                    try (BufferedReader reader = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8.name()))) {
                        String[] columnHeaders = null;
                        String line = reader.readLine();
                        while (line != null) {
                            String[] fields = line.split(SeparatorConstants.TAB);
                            if (columnHeaders == null) {
                                columnHeaders = fields;
                            } else {
                                recordStore.add();
                                final int fieldsCount = Math.min(columnHeaders.length, fields.length);
                                for (int i = 0; i < fieldsCount; i++) {
                                    recordStore.set(columnHeaders[i], fields[i]);
                                }
                            }
                            line = reader.readLine();
                        }
                    }
                // If any directories are encountered then don't allow the drop
                } else {
                    badData = true;
                    break;
                }
            }
            if (!badData && recordStore.size() > 0) {
                return (graph, dropInfo) -> PluginExecution.withPlugin(new TSVDropperToGraphPlugin(recordStore, files)).executeLater(graph);
            }
        } catch (final UnsupportedFlavorException | IOException ex) {
            Exceptions.printStackTrace(ex);
        }
    }
    return null;
}

Example 20 with GraphRecordStore

use of au.gov.asd.tac.constellation.graph.processing.GraphRecordStore in project constellation by constellation-app.

the class WorkflowQueryPlugin method execute.

@Override
protected void execute(final PluginGraphs pluginGraphs, final PluginInteraction interaction, final PluginParameters parameters) throws InterruptedException, PluginException {
    final Graph graph = pluginGraphs.getGraph();
    final ReadableGraph readableGraph = graph.getReadableGraph();
    // buildId batches
    try {
        queryBatches = GraphRecordStoreUtilities.getSelectedVerticesBatches(readableGraph, parameters.getIntegerValue(BATCH_SIZE_PARAMETER_ID));
    } finally {
        readableGraph.release();
    }
    pluginGraphs.waitAtGate(1);
    // create a service for executing jobs, limiting concurrent executions to the max concurrent plugins parameter.
    final int maxConcurrentPlugins = parameters.getIntegerValue(MAX_CONCURRENT_PLUGINS_PARAMETER_ID);
    final ExecutorService workflowExecutor = Executors.newFixedThreadPool(maxConcurrentPlugins);
    // schedule a job for each batch, where the job is to execute the defined workflow
    final List<Future<?>> workerPlugins = new ArrayList<>();
    final List<PluginException> exceptions = new ArrayList<>();
    if (queryBatches.isEmpty()) {
        queryBatches.add(new GraphRecordStore());
    }
    // run plugin once for every batch record store
    queryBatches.forEach(batch -> {
        final StoreGraph batchGraph = new StoreGraph(graph.getSchema() != null ? graph.getSchema().getFactory().createSchema() : null);
        batchGraph.getSchema().newGraph(batchGraph);
        CopyGraphUtilities.copyGraphTypeElements(readableGraph, batchGraph);
        GraphRecordStoreUtilities.addRecordStoreToGraph(batchGraph, batch, true, true, null);
        final WorkerQueryPlugin worker = new WorkerQueryPlugin(getWorkflow(), batchGraph, exceptions, getErrorHandlingPlugin(), addPartialResults());
        workerPlugins.add(workflowExecutor.submit(() -> {
            Thread.currentThread().setName(THREAD_POOL_NAME);
            try {
                PluginExecution.withPlugin(worker).withParameters(parameters).executeNow(graph);
            } catch (InterruptedException | PluginException ex) {
                throw new RuntimeException(ex);
            }
        }));
    });
    final int[] workerFailCount = { 0 };
    for (Future<?> worker : workerPlugins) {
        try {
            worker.get();
        } catch (InterruptedException ex) {
            workerPlugins.forEach(workerToInterrupt -> workerToInterrupt.cancel(true));
            throw ex;
        } catch (ExecutionException ex) {
            workerFailCount[0]++;
        }
    }
    workflowExecutor.shutdown();
    // if there were any errors, collect them and display them to the user
    if (!exceptions.isEmpty()) {
        final StringBuilder entireException = new StringBuilder();
        entireException.append(workerFailCount[0]).append(" workers failed.").append(SeparatorConstants.NEWLINE);
        exceptions.forEach(ex -> entireException.append(ex.getMessage()).append(SeparatorConstants.NEWLINE));
        throw new PluginException(PluginNotificationLevel.ERROR, entireException.toString());
    }
}