Examples with ImportAttributeDefinition au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition used on opensource projects

Example 1 with ImportAttributeDefinition

use of au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition in project constellation by constellation-app.

the class ImportDelimitedPlugin method processTransactions.

private static int processTransactions(ImportDefinition definition, GraphWriteMethods graph, List<String[]> data, boolean initialiseWithSchema, PluginInteraction interaction, String source) throws InterruptedException {
    final List<ImportAttributeDefinition> sourceVertexDefinitions = definition.getDefinitions(AttributeType.SOURCE_VERTEX);
    final List<ImportAttributeDefinition> destinationVertexDefinitions = definition.getDefinitions(AttributeType.DESTINATION_VERTEX);
    final List<ImportAttributeDefinition> transactionDefinitions = definition.getDefinitions(AttributeType.TRANSACTION);
    int directedIx = ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN;
    for (int i = 0; i < transactionDefinitions.size(); i++) {
        if (transactionDefinitions.get(i).getAttribute().getName().equals(DelimitedImportController.DIRECTED)) {
            directedIx = transactionDefinitions.get(i).getColumnIndex();
            break;
        }
    }
    addAttributes(graph, GraphElementType.VERTEX, sourceVertexDefinitions);
    addAttributes(graph, GraphElementType.VERTEX, destinationVertexDefinitions);
    addAttributes(graph, GraphElementType.TRANSACTION, transactionDefinitions);
    int currentRow = 0;
    int importedRows = 0;
    final int totalRows = data.size() - definition.getFirstRow();
    final RowFilter filter = definition.getRowFilter();
    for (int i = definition.getFirstRow(); i < data.size(); i++) {
        interaction.setProgress(++currentRow, totalRows, "Importing Transactions: " + source, true);
        final String[] row = data.get(i);
        if (filter == null || filter.passesFilter(i - 1, row)) {
            // Count the number of processed rows to notify in the status message
            ++importedRows;
            final int sourceVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : sourceVertexDefinitions) {
                attributeDefinition.setValue(graph, sourceVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, sourceVertexId);
            }
            final int destinationVertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : destinationVertexDefinitions) {
                attributeDefinition.setValue(graph, destinationVertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, destinationVertexId);
            }
            final boolean isDirected = directedIx == ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN || Boolean.parseBoolean(row[directedIx]);
            final int transactionId = graph.addTransaction(sourceVertexId, destinationVertexId, isDirected);
            for (final ImportAttributeDefinition attributeDefinition : transactionDefinitions) {
                if (attributeDefinition.getOverriddenAttributeId() != Graph.NOT_FOUND) {
                    attributeDefinition.setValue(graph, transactionId, row, (i - 1));
                }
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeTransaction(graph, transactionId);
            }
        }
    }
    return importedRows;
}

Example 2 with ImportAttributeDefinition

use of au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition in project constellation by constellation-app.

the class ImportJDBCIO method loadParameterFile.

private static void loadParameterFile(final JDBCImportController importController, final File delimIoDir, final String templName) {
    try {
        final ObjectMapper mapper = new ObjectMapper();
        final JsonNode root = mapper.readTree(new File(delimIoDir, FilenameEncoder.encode(templName) + FileExtensionConstants.JSON));
        final JsonNode source = root.get(SOURCE);
        final boolean schemaInit = source.get(SCHEMA_INIT).booleanValue();
        importController.setSchemaInitialised(schemaInit);
        final String destination = source.get(DESTINATION).textValue();
        final SchemaFactory schemaFactory = SchemaFactoryUtilities.getSchemaFactory(destination);
        if (schemaFactory != null) {
            importController.setDestination(new SchemaDestination(schemaFactory));
            final List<ImportDefinition> definitions = new ArrayList<>();
            final ArrayNode definitionsArray = (ArrayNode) root.withArray(DEFINITIONS);
            for (final JsonNode definitionNode : definitionsArray) {
                final int firstRow = definitionNode.get(FIRST_ROW).intValue();
                final RowFilter filter = new RowFilter();
                if (definitionNode.has(FILTER)) {
                    final JsonNode filterNode = definitionNode.get(FILTER);
                    final String script = filterNode.get(SCRIPT).textValue();
                    final JsonNode columnsArray = filterNode.withArray(COLUMNS);
                    final List<String> columns = new ArrayList<>();
                    for (final JsonNode column : columnsArray) {
                        columns.add(column.textValue());
                    }
                    filter.setScript(script);
                    filter.setColumns(columns.toArray(new String[columns.size()]));
                }
                final ImportDefinition impdef = new ImportDefinition("", firstRow, filter);
                final JsonNode attributesNode = definitionNode.get(ATTRIBUTES);
                for (final AttributeType attrType : AttributeType.values()) {
                    final ArrayNode columnArray = (ArrayNode) attributesNode.withArray(attrType.toString());
                    for (final JsonNode column : columnArray) {
                        final String columnLabel = column.get(COLUMN_LABEL).textValue();
                        final String label = column.get(ATTRIBUTE_LABEL).textValue();
                        if (!importController.hasAttribute(attrType.getElementType(), label)) {
                            // Manually created attribute.
                            final String type = column.get(ATTRIBUTE_TYPE).textValue();
                            final String descr = column.get(ATTRIBUTE_DESCRIPTION).textValue();
                            final NewAttribute a = new NewAttribute(attrType.getElementType(), type, label, descr);
                            importController.createManualAttribute(a);
                        }
                        final Attribute attribute = importController.getAttribute(attrType.getElementType(), label);
                        final AttributeTranslator translator = AttributeTranslator.getTranslator(column.get(TRANSLATOR).textValue());
                        final String args = column.get(TRANSLATOR_ARGS).textValue();
                        final String defaultValue = column.get(DEFAULT_VALUE).textValue();
                        final PluginParameters params = translator.createParameters();
                        translator.setParameterValues(params, args);
                        final ImportAttributeDefinition iad = new ImportAttributeDefinition(columnLabel, defaultValue, attribute, translator, params);
                        impdef.addDefinition(attrType, iad);
                    }
                }
                definitions.add(impdef);
            }
            importController.setClearManuallyAdded(false);
            try {
                ((JDBCImportPane) importController.getStage()).update(importController, definitions);
            } finally {
                importController.setClearManuallyAdded(true);
            }
        } else {
            final String msg = String.format("Can't find schema factory '%s'", destination);
            final NotifyDescriptor nd = new NotifyDescriptor.Message(msg, NotifyDescriptor.ERROR_MESSAGE);
            DialogDisplayer.getDefault().notify(nd);
        }
    } catch (final IOException ex) {
        LOGGER.log(Level.SEVERE, ex.getLocalizedMessage(), ex);
    }
}

Example 3 with ImportAttributeDefinition

use of au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition in project constellation by constellation-app.

the class ImportJDBCIO method definitionCompute.

private static void definitionCompute(final ArrayNode definitionArray, final String[] columns, final ImportDefinition impdef) {
    final ObjectNode def = definitionArray.addObject();
    def.put(FIRST_ROW, impdef.getFirstRow());
    if (impdef.getRowFilter() != null) {
        final ObjectNode filter = def.putObject(FILTER);
        final ArrayNode columnsArray = filter.putArray(COLUMNS);
        for (final String column : impdef.getRowFilter().getColumns()) {
            columnsArray.add(column);
        }
        final String script = impdef.getRowFilter().getScript();
        if (script != null) {
            filter.put(SCRIPT, script);
        } else {
            filter.putNull(SCRIPT);
        }
    }
    final ObjectNode attrDefs = def.putObject(ATTRIBUTES);
    for (final AttributeType attrType : AttributeType.values()) {
        final ArrayNode typeArray = attrDefs.putArray(attrType.name());
        final List<ImportAttributeDefinition> iadefs = impdef.getDefinitions(attrType);
        final Consumer<ImportAttributeDefinition> importConsumer = (ImportAttributeDefinition iadef) -> {
            if (hasSavableAttribute(iadef)) {
                final ObjectNode type = typeArray.addObject();
                // that the settings still get applied in the attribute list on load
                if (iadef.getColumnIndex() == ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN) {
                    type.putNull(COLUMN_LABEL);
                } else {
                    type.put(COLUMN_LABEL, columns[iadef.getColumnIndex() + 1]);
                }
                type.put(ATTRIBUTE_LABEL, iadef.getAttribute().getName());
                if (iadef.getAttribute() instanceof NewAttribute) {
                    type.put(ATTRIBUTE_TYPE, iadef.getAttribute().getAttributeType());
                    type.put(ATTRIBUTE_DESCRIPTION, iadef.getAttribute().getDescription());
                }
                type.put(TRANSLATOR, iadef.getTranslator().getClass().getName());
                type.put(TRANSLATOR_ARGS, iadef.getTranslator().getParameterValues(iadef.getParameters()));
                if (iadef.getDefaultValue() != null) {
                    type.put(DEFAULT_VALUE, iadef.getDefaultValue());
                } else {
                    type.putNull(DEFAULT_VALUE);
                }
            }
        };
        iadefs.stream().forEach(importConsumer);
    }
}

Example 4 with ImportAttributeDefinition

use of au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition in project constellation by constellation-app.

the class ImportJDBCPlugin method processVertices.

private static int processVertices(final ImportDefinition definition, final GraphWriteMethods graph, final List<String[]> data, final AttributeType attributeType, final boolean initialiseWithSchema, final PluginInteraction interaction) throws InterruptedException {
    final List<ImportAttributeDefinition> attributeDefinitions = definition.getDefinitions(attributeType);
    addAttributes(graph, GraphElementType.VERTEX, attributeDefinitions);
    int currentRow = 0;
    int importedRows = 0;
    final int totalRows = data.size() - definition.getFirstRow();
    final RowFilter filter = definition.getRowFilter();
    for (int i = definition.getFirstRow(); i < data.size(); i++) {
        interaction.setProgress(++currentRow, totalRows, "Importing Vertices", true);
        final String[] row = data.get(i);
        if (filter == null || filter.passesFilter(i, row)) {
            // Count the number of processed rows to notify in the status message
            ++importedRows;
            final int vertexId = graph.addVertex();
            for (final ImportAttributeDefinition attributeDefinition : attributeDefinitions) {
                attributeDefinition.setValue(graph, vertexId, row, (i - 1));
            }
            if (initialiseWithSchema && graph.getSchema() != null) {
                graph.getSchema().completeVertex(graph, vertexId);
            }
        }
    }
    return importedRows;
}

Example 5 with ImportAttributeDefinition

use of au.gov.asd.tac.constellation.plugins.importexport.ImportAttributeDefinition in project constellation by constellation-app.

the class ImportDelimitedIO method saveParameters.

public static void saveParameters(final Window parentWindow, final DelimitedImportController importController) {
    final Preferences prefs = NbPreferences.forModule(ApplicationPreferenceKeys.class);
    final String userDir = ApplicationPreferenceKeys.getUserDir(prefs);
    final File delimIoDir = new File(userDir, IMPORT_DELIMITED_DIR);
    if (!delimIoDir.exists()) {
        delimIoDir.mkdir();
    }
    if (!delimIoDir.isDirectory()) {
        final String message = String.format("Can't create directory '%s'.", delimIoDir);
        NotifyDisplayer.displayAlert(SAVE_TEMPLATE, "Templates Directory Error", message, Alert.AlertType.ERROR);
        return;
    }
    final String templName = new TemplateListDialog(parentWindow, false).getName(delimIoDir);
    if (templName != null) {
        // A JSON document to store everything in.
        // Two objects; the source data + the configuration data.
        final ObjectMapper mapper = new ObjectMapper();
        final ObjectNode rootNode = mapper.createObjectNode();
        final ObjectNode source = rootNode.putObject(SOURCE);
        source.put(PARSER, importController.getImportFileParser().getLabel());
        source.put(SCHEMA_INIT, importController.isSchemaInitialised());
        source.put(SHOW_ALL_SCHEMA_ATTRIBUTES, importController.isShowAllSchemaAttributesEnabled());
        source.put(FILES_INCLUDE_HEADERS, importController.isFilesIncludeHeadersEnabled());
        // We don't want to rely on a particular kind of graph being current when we load this definition.
        // Therefore, we only save a schema factory as the destination.
        final ImportDestination<?> importd = importController.getDestination();
        final String destination;
        if (importd instanceof SchemaDestination) {
            destination = ((SchemaDestination) importd).getDestination().getName();
        } else if (importd instanceof GraphDestination) {
            destination = ((GraphDestination) importd).getGraph().getSchema().getFactory().getName();
        } else {
            throw new IllegalArgumentException(String.format("Unrecognised destination '%s'", importd));
        }
        source.put(DESTINATION, destination);
        // One ImportDefinition per tab.
        final ArrayNode definitionArray = rootNode.putArray(DEFINITIONS);
        final List<ImportDefinition> definitions = importController.getDefinitions(importController.isFilesIncludeHeadersEnabled());
        final String[] columns = importController.getCurrentColumns();
        final Consumer<ImportDefinition> definitionConsumer = (ImportDefinition impdef) -> {
            final ObjectNode def = definitionArray.addObject();
            def.put(FIRST_ROW, impdef.getFirstRow());
            if (impdef.getRowFilter() != null) {
                final ObjectNode filter = def.putObject(FILTER);
                final ArrayNode columnsArray = filter.putArray(COLUMNS);
                for (final String column : impdef.getRowFilter().getColumns()) {
                    columnsArray.add(column);
                }
                final String script = impdef.getRowFilter().getScript();
                if (script != null) {
                    filter.put(SCRIPT, script);
                } else {
                    filter.putNull(SCRIPT);
                }
            }
            final ObjectNode attrDefs = def.putObject(ATTRIBUTES);
            for (final AttributeType attrType : AttributeType.values()) {
                final ArrayNode typeArray = attrDefs.putArray(attrType.name());
                final List<ImportAttributeDefinition> iadefs = impdef.getDefinitions(attrType);
                final Consumer<ImportAttributeDefinition> attributeDefConsumer = (ImportAttributeDefinition iadef) -> {
                    if (hasSavableAttribute(iadef)) {
                        final ObjectNode type = typeArray.addObject();
                        // that the settings still get applied in the attribute list on load
                        if (iadef.getColumnIndex() == ImportConstants.ATTRIBUTE_NOT_ASSIGNED_TO_COLUMN) {
                            type.putNull(COLUMN_LABEL);
                        } else {
                            type.put(COLUMN_LABEL, columns[iadef.getColumnIndex() + 1]);
                        }
                        type.put(ATTRIBUTE_LABEL, iadef.getAttribute().getName());
                        if (iadef.getAttribute() instanceof NewAttribute) {
                            type.put(ATTRIBUTE_TYPE, iadef.getAttribute().getAttributeType());
                            type.put(ATTRIBUTE_DESCRIPTION, iadef.getAttribute().getDescription());
                        }
                        type.put(TRANSLATOR, iadef.getTranslator().getClass().getName());
                        type.put(TRANSLATOR_ARGS, iadef.getTranslator().getParameterValues(iadef.getParameters()));
                        if (iadef.getDefaultValue() != null) {
                            type.put(DEFAULT_VALUE, iadef.getDefaultValue());
                        } else {
                            type.putNull(DEFAULT_VALUE);
                        }
                    }
                };
                iadefs.stream().forEach(attributeDefConsumer);
            }
        };
        definitions.stream().forEach(definitionConsumer);
        mapper.configure(SerializationFeature.INDENT_OUTPUT, true);
        mapper.configure(SerializationFeature.CLOSE_CLOSEABLE, true);
        final File f = new File(delimIoDir, FilenameEncoder.encode(templName + FileExtensionConstants.JSON));
        try {
            mapper.writeValue(f, rootNode);
            StatusDisplayer.getDefault().setStatusText(String.format("Import definition saved to %s.", f.getPath()));
        } catch (final IOException ex) {
            NotifyDisplayer.display(String.format("Can't save import definition: %s", ex.getMessage()), NotifyDescriptor.ERROR_MESSAGE);
            LOGGER.log(Level.WARNING, ex.getMessage(), ex);
        }
    }
}