Examples with RowFilter au.gov.asd.tac.constellation.plugins.importexport.RowFilter used on opensource projects

Example 1 with RowFilter

use of au.gov.asd.tac.constellation.plugins.importexport.RowFilter in project constellation by constellation-app.

the class ImportDelimitedPlugin method processTransactions.

private static int processTransactions(ImportDefinition definition, GraphWriteMethods graph, List<String[]> data, boolean initialiseWithSchema, PluginInteraction interaction, String source) throws InterruptedException {
    final List<ImportAttributeDefinition> sourceVertexDefinitions = definition.getDefinitions(AttributeType.SOURCE_VERTEX);
    final List<ImportAttributeDefinition> destinationVertexDefinitions = definition.getDefinitions(AttributeType.DESTINATION_VERTEX);
    final List<ImportAttributeDefinition> transactionDefinitions = definition.getDefinitions(AttributeType.TRANSACTION);
    int directedIx = ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN;
    for (int i = 0; i < transactionDefinitions.size(); i++) {
        if (transactionDefinitions.get(i).getAttribute().getName().equals(DelimitedImportController.DIRECTED)) {
            directedIx = transactionDefinitions.get(i).getColumnIndex();
            break;
        }
    }
    addAttributes(graph, GraphElementType.VERTEX, sourceVertexDefinitions);
    addAttributes(graph, GraphElementType.VERTEX, destinationVertexDefinitions);
    addAttributes(graph, GraphElementType.TRANSACTION, transactionDefinitions);
    int currentRow = 0;
    int importedRows = 0;
    final int totalRows = data.size() - definition.getFirstRow();
    final RowFilter filter = definition.getRowFilter();
    for (int i = definition.getFirstRow(); i < data.size(); i++) {
        interaction.setProgress(++currentRow, totalRows, "Importing Transactions: " + source, true);
        final String[] row = data.get(i);
        if (filter == null || filter.passesFilter(i - 1, row)) {
            // Count the number of processed rows to notify in the status message
            ++importedRows;
            final int sourceVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : sourceVertexDefinitions) {
                attributeDefinition.setValue(graph, sourceVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, sourceVertexId);
            }
            final int destinationVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : destinationVertexDefinitions) {
                attributeDefinition.setValue(graph, destinationVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, destinationVertexId);
            }
            final boolean isDirected = directedIx == ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN || Boolean.parseBoolean(row[directedIx]);
            final int transactionId = graph.addTransaction(sourceVertexId, destinationVertexId, isDirected);
            for (final ImportAttributeDefinition attributeDefinition : transactionDefinitions) {
                if (attributeDefinition.getOverriddenAttributeId() != Graph.NOT_FOUND) {
                    attributeDefinition.setValue(graph, transactionId, row, (i - 1));
                }
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeTransaction(graph, transactionId);
            }
        }
    }
    return importedRows;
}

Example 2 with RowFilter

use of au.gov.asd.tac.constellation.plugins.importexport.RowFilter in project constellation by constellation-app.

the class ImportJDBCIO method loadParameterFile.

private static void loadParameterFile(final JDBCImportController importController, final File delimIoDir, final String templName) {
    try {
        final ObjectMapper mapper = new ObjectMapper();
        final JsonNode root = mapper.readTree(new File(delimIoDir, FilenameEncoder.encode(templName) + FileExtensionConstants.JSON));
        final JsonNode source = root.get(SOURCE);
        final boolean schemaInit = source.get(SCHEMA_INIT).booleanValue();
        importController.setSchemaInitialised(schemaInit);
        final String destination = source.get(DESTINATION).textValue();
        final SchemaFactory schemaFactory = SchemaFactoryUtilities.getSchemaFactory(destination);
        if (schemaFactory != null) {
            importController.setDestination(new SchemaDestination(schemaFactory));
            final List<ImportDefinition> definitions = new ArrayList<>();
            final ArrayNode definitionsArray = (ArrayNode) root.withArray(DEFINITIONS);
            for (final JsonNode definitionNode : definitionsArray) {
                final int firstRow = definitionNode.get(FIRST_ROW).intValue();
                final RowFilter filter = new RowFilter();
                if (definitionNode.has(FILTER)) {
                    final JsonNode filterNode = definitionNode.get(FILTER);
                    final String script = filterNode.get(SCRIPT).textValue();
                    final JsonNode columnsArray = filterNode.withArray(COLUMNS);
                    final List<String> columns = new ArrayList<>();
                    for (final JsonNode column : columnsArray) {
                        columns.add(column.textValue());
                    }
                    filter.setScript(script);
                    filter.setColumns(columns.toArray(new String[columns.size()]));
                }
                final ImportDefinition impdef = new ImportDefinition("", firstRow, filter);
                final JsonNode attributesNode = definitionNode.get(ATTRIBUTES);
                for (final AttributeType attrType : AttributeType.values()) {
                    final ArrayNode columnArray = (ArrayNode) attributesNode.withArray(attrType.toString());
                    for (final JsonNode column : columnArray) {
                        final String columnLabel = column.get(COLUMN_LABEL).textValue();
                        final String label = column.get(ATTRIBUTE_LABEL).textValue();
                        if (!importController.hasAttribute(attrType.getElementType(), label)) {
                            // Manually created attribute.
                            final String type = column.get(ATTRIBUTE_TYPE).textValue();
                            final String descr = column.get(ATTRIBUTE_DESCRIPTION).textValue();
                            final NewAttribute a = new NewAttribute(attrType.getElementType(), type, label, descr);
                            importController.createManualAttribute(a);
                        }
                        final Attribute attribute = importController.getAttribute(attrType.getElementType(), label);
                        final AttributeTranslator translator = AttributeTranslator.getTranslator(column.get(TRANSLATOR).textValue());
                        final String args = column.get(TRANSLATOR_ARGS).textValue();
                        final String defaultValue = column.get(DEFAULT_VALUE).textValue();
                        final PluginParameters params = translator.createParameters();
                        translator.setParameterValues(params, args);
                        final ImportAttributeDefinition iad = new ImportAttributeDefinition(columnLabel, defaultValue, attribute, translator, params);
                        impdef.addDefinition(attrType, iad);
                    }
                }
                definitions.add(impdef);
            }
            importController.setClearManuallyAdded(false);
            try {
                ((JDBCImportPane) importController.getStage()).update(importController, definitions);
            } finally {
                importController.setClearManuallyAdded(true);
            }
        } else {
            final String msg = String.format("Can't find schema factory '%s'", destination);
            final NotifyDescriptor nd = new NotifyDescriptor.Message(msg, NotifyDescriptor.ERROR_MESSAGE);
            DialogDisplayer.getDefault().notify(nd);
        }
    } catch (final IOException ex) {
        LOGGER.log(Level.SEVERE, ex.getLocalizedMessage(), ex);
    }
}

Example 3 with RowFilter

use of au.gov.asd.tac.constellation.plugins.importexport.RowFilter in project constellation by constellation-app.

the class ImportJDBCPlugin method processVertices.

private static int processVertices(final ImportDefinition definition, final GraphWriteMethods graph, final List<String[]> data, final AttributeType attributeType, final boolean initialiseWithSchema, final PluginInteraction interaction) throws InterruptedException {
    final List<ImportAttributeDefinition> attributeDefinitions = definition.getDefinitions(attributeType);
    addAttributes(graph, GraphElementType.VERTEX, attributeDefinitions);
    int currentRow = 0;
    int importedRows = 0;
    final int totalRows = data.size() - definition.getFirstRow();
    final RowFilter filter = definition.getRowFilter();
    for (int i = definition.getFirstRow(); i < data.size(); i++) {
        interaction.setProgress(++currentRow, totalRows, "Importing Vertices", true);
        final String[] row = data.get(i);
        if (filter == null || filter.passesFilter(i, row)) {
            // Count the number of processed rows to notify in the status message
            ++importedRows;
            final int vertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : attributeDefinitions) {
                attributeDefinition.setValue(graph, vertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, vertexId);
            }
        }
    }
    return importedRows;
}

Example 4 with RowFilter

use of au.gov.asd.tac.constellation.plugins.importexport.RowFilter in project constellation by constellation-app.

the class ImportDelimitedIO method loadParameterFile.

private static void loadParameterFile(final DelimitedImportController importController, final File delimIoDir, final String templName) {
    try {
        final ObjectMapper mapper = new ObjectMapper();
        final JsonNode root = mapper.readTree(new File(delimIoDir, FilenameEncoder.encode(templName) + FileExtensionConstants.JSON));
        final JsonNode source = root.get(SOURCE);
        final String parser = source.get(PARSER).textValue();
        final ImportFileParser ifp = ImportFileParser.getParser(parser);
        if (!importController.getImportFileParser().getLabel().equals(parser)) {
            final String message = String.format("Template is for a different file Parser '%s'.", parser);
            NotifyDisplayer.displayAlert(LOAD_TEMPLATE, "File Parser Mismatch", message, Alert.AlertType.ERROR);
            return;
        }
        importController.setImportFileParser(ifp);
        final boolean schemaInit = source.get(SCHEMA_INIT).booleanValue();
        importController.setSchemaInitialised(schemaInit);
        final boolean filesIncludeHeaders = source.get(FILES_INCLUDE_HEADERS).booleanValue();
        importController.setfilesIncludeHeaders(filesIncludeHeaders);
        final boolean showAllSchemaAttributes = source.get(SHOW_ALL_SCHEMA_ATTRIBUTES) != null && source.get(SHOW_ALL_SCHEMA_ATTRIBUTES).booleanValue();
        importController.setShowAllSchemaAttributes(showAllSchemaAttributes);
        final String destination = source.get(DESTINATION).textValue();
        final SchemaFactory schemaFactory = SchemaFactoryUtilities.getSchemaFactory(destination);
        if (schemaFactory != null) {
            importController.setDestination(new SchemaDestination(schemaFactory));
            final List<ImportDefinition> definitions = new ArrayList<>();
            final ArrayNode definitionsArray = (ArrayNode) root.withArray(DEFINITIONS);
            for (final JsonNode definitionNode : definitionsArray) {
                final int firstRow = definitionNode.get(FIRST_ROW).intValue();
                final RowFilter filter = new RowFilter();
                if (definitionNode.has(FILTER)) {
                    final JsonNode filterNode = definitionNode.get(FILTER);
                    final String script = filterNode.get(SCRIPT).textValue();
                    final JsonNode columnsArray = filterNode.withArray(COLUMNS);
                    final ArrayList<String> columns = new ArrayList<>();
                    for (final JsonNode column : columnsArray) {
                        columns.add(column.textValue());
                    }
                    filter.setScript(script);
                    filter.setColumns(columns.toArray(new String[columns.size()]));
                }
                final ImportDefinition impdef = new ImportDefinition("", firstRow, filter);
                final JsonNode attributesNode = definitionNode.get(ATTRIBUTES);
                for (final AttributeType attrType : AttributeType.values()) {
                    final ArrayNode columnArray = (ArrayNode) attributesNode.withArray(attrType.toString());
                    for (final JsonNode column : columnArray) {
                        final String columnLabel = column.get(COLUMN_LABEL).textValue();
                        final String label = column.get(ATTRIBUTE_LABEL).textValue();
                        if (!importController.hasAttribute(attrType.getElementType(), label)) {
                            // Manually created attribute.
                            final String type = column.get(ATTRIBUTE_TYPE).textValue();
                            final String descr = column.get(ATTRIBUTE_DESCRIPTION).textValue();
                            final NewAttribute a = new NewAttribute(attrType.getElementType(), type, label, descr);
                            importController.createManualAttribute(a);
                        }
                        final Attribute attribute = importController.getAttribute(attrType.getElementType(), label);
                        final AttributeTranslator translator = AttributeTranslator.getTranslator(column.get(TRANSLATOR).textValue());
                        final String args = column.get(TRANSLATOR_ARGS).textValue();
                        final String defaultValue = column.get(DEFAULT_VALUE).textValue();
                        final PluginParameters params = translator.createParameters();
                        translator.setParameterValues(params, args);
                        final ImportAttributeDefinition iad = new ImportAttributeDefinition(columnLabel, defaultValue, attribute, translator, params);
                        impdef.addDefinition(attrType, iad);
                    }
                }
                definitions.add(impdef);
            }
            importController.setClearManuallyAdded(false);
            try {
                ((DelimitedImportPane) importController.getStage()).update(importController, definitions);
            } finally {
                importController.setClearManuallyAdded(true);
            }
        } else {
            final String message = String.format("Can't find schema factory '%s'", destination);
            NotifyDisplayer.displayAlert(LOAD_TEMPLATE, "Destination Schema Error", message, Alert.AlertType.ERROR);
        }
    } catch (final IOException ex) {
        LOGGER.log(Level.SEVERE, ex.getLocalizedMessage(), ex);
    }
}

Example 5 with RowFilter

use of au.gov.asd.tac.constellation.plugins.importexport.RowFilter in project constellation by constellation-app.

the class ImportJDBCPlugin method processTransactions.

private static int processTransactions(final ImportDefinition definition, final GraphWriteMethods graph, final List<String[]> data, final boolean initialiseWithSchema, final PluginInteraction interaction) throws InterruptedException {
    final List<ImportAttributeDefinition> sourceVertexDefinitions = definition.getDefinitions(AttributeType.SOURCE_VERTEX);
    final List<ImportAttributeDefinition> destinationVertexDefinitions = definition.getDefinitions(AttributeType.DESTINATION_VERTEX);
    final List<ImportAttributeDefinition> transactionDefinitions = definition.getDefinitions(AttributeType.TRANSACTION);
    int directedIx = ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN;
    for (int i = 0; i < transactionDefinitions.size(); i++) {
        if (transactionDefinitions.get(i).getAttribute().getName().equals(JDBCImportController.DIRECTED)) {
            directedIx = transactionDefinitions.get(i).getColumnIndex();
            break;
        }
    }
    addAttributes(graph, GraphElementType.VERTEX, sourceVertexDefinitions);
    addAttributes(graph, GraphElementType.VERTEX, destinationVertexDefinitions);
    addAttributes(graph, GraphElementType.TRANSACTION, transactionDefinitions);
    int currentRow = 0;
    int importedRows = 0;
    final int totalRows = data.size() - definition.getFirstRow();
    final RowFilter filter = definition.getRowFilter();
    for (int i = definition.getFirstRow(); i < data.size(); i++) {
        interaction.setProgress(++currentRow, totalRows, "Importing Transactions", true);
        final String[] row = data.get(i);
        if (filter == null || filter.passesFilter(i, row)) {
            // Count the number of processed rows to notify in the status message
            ++importedRows;
            final int sourceVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : sourceVertexDefinitions) {
                attributeDefinition.setValue(graph, sourceVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, sourceVertexId);
            }
            final int destinationVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : destinationVertexDefinitions) {
                attributeDefinition.setValue(graph, destinationVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, destinationVertexId);
            }
            final boolean isDirected = directedIx == ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN || Boolean.parseBoolean(row[directedIx]);
            final int transactionId = graph.addTransaction(sourceVertexId, destinationVertexId, isDirected);
            for (final ImportAttributeDefinition attributeDefinition : transactionDefinitions) {
                if (attributeDefinition.getOverriddenAttributeId() != Graph.NOT_FOUND) {
                    attributeDefinition.setValue(graph, transactionId, row, (i - 1));
                }
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeTransaction(graph, transactionId);
            }
        }
    }
    return importedRows;
}