Sync interface, assay now incremental for adding other packages, name in sheetname and export fixes

package.json

@@ -68,7 +68,6 @@
     "lit-element": "2.5.1",
     "lit-html": "1.4.1",
     "terser": "4.6.7"
-
   },
   "devDependencies": {
     "babel-loader": "8.2.2",
@@ -178,6 +177,6 @@
       "webpack-dev-server": "3.11.0",
       "webpack-merge": "4.2.2"
     },
-    "hash": "c026424a89aa150aada71c0fce2135327cdd92025da4c8a55e6453aff50825d0"
+    "hash": "8587df101e8898757cb19c3eb0932ffc91999721c9c0def3702d8a367cdd6e1f"
   }
 }

src/main/java/nl/fairbydesign/backend/metadata/MetadataParser.java

@@ -138,7 +138,7 @@ public class MetadataParser extends Metadata {
         // email value from https://www.emailregex.com
         syntax = syntax.replace("{email}", "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])");
         // orcid regex
-        syntax = syntax.replace("{orcid}", "(\\d|X{4}-){3}\\d|X{4}$");
+        syntax = syntax.replace("{orcid}", "^(\\d|X{4}-){3}\\d|X{4}$");
         // Day(s) unit
         syntax = syntax.replace("{day}", "day[s]?");
         // Units are to broad so it becomes free text
@@ -284,7 +284,7 @@ public class MetadataParser extends Metadata {
         SortedMap<String, ArrayList<Metadata>> sheetObjects = new TreeMap<>();
         for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
             Sheet sheet = workbook.getSheetAt(i);
-            logger.debug("Processing " + sheet.getSheetName() + " " + sheet.getLastRowNum());
+            logger.info("Processing " + sheet.getSheetName() + " " + sheet.getLastRowNum());
             String name = sheet.getSheetName();
             ArrayList<Metadata> metadataHashSet = MetadataParser.parser(sheet);
             sheetObjects.put(name, metadataHashSet);

src/main/java/nl/fairbydesign/backend/parsers/ExcelGenerator.java

@@ -44,9 +44,9 @@ public class ExcelGenerator {
         workbook = new XSSFWorkbook();
 
         // set sample package property
-        POIXMLProperties props = workbook.getProperties();
-        POIXMLProperties.CustomProperties custProp = props.getCustomProperties();
-        custProp.addProperty("sample", samplePackage);
+//        POIXMLProperties props = workbook.getProperties();
+//        POIXMLProperties.CustomProperties custProp = props.getCustomProperties();
+        //custProp.addProperty("sample", samplePackage);
 
         // Create excel style object
         makeHeaderCellStyle();
@@ -72,21 +72,28 @@ public class ExcelGenerator {
         for (String key : gridSelection.keySet()) {
             if (key.toLowerCase().contains("sample")) {
                 // Overwrite key name by including package name
-                String sheetName = key + " - " + samplePackage;
-                generateAssaySheet(workbook, sheetName, gridSelection.get(key));
+                generateAssaySheet(workbook, key, gridSelection.get(key));
+                remove.add(key);
+            }
+        }
+
+        // Create assays, all for loops are separated to preserve order
+        for (String key : gridSelection.keySet()) {
+            if (key.toLowerCase().contains("assay")) {
+                // Overwrite key name by including package name
+                generateAssaySheet(workbook, key, gridSelection.get(key));
                 remove.add(key);
-                break;
             }
         }
 
         // Removing the observation unit and sample object
-        gridSelection.keySet().removeAll(remove);
+//        gridSelection.keySet().removeAll(remove);
 
         // Perform the rest of the assays
-        for (String key : gridSelection.keySet()) {
-            logger.info("Creating assay sheet " + key);
-            generateAssaySheet(workbook, key, gridSelection.get(key));
-        }
+//        for (String key : gridSelection.keySet()) {
+//            logger.info("Creating assay sheet " + key);
+//            generateAssaySheet(workbook, key, gridSelection.get(key));
+//        }
 
         // Save file and escape the / in a name
         File filename = new File(project.getIdentifier().replaceAll("/","-") + ".xlsx");

src/main/java/nl/fairbydesign/backend/parsers/ExcelValidator.java

@@ -11,6 +11,9 @@ import nl.wur.ssb.RDFSimpleCon.api.Domain;
 import org.apache.commons.codec.digest.DigestUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.jena.rdf.model.NodeIterator;
+import org.apache.jena.rdf.model.RDFList;
+import org.apache.jena.rdf.model.Resource;
+import org.apache.jena.rdf.model.StmtIterator;
 import org.apache.poi.hssf.usermodel.HSSFDateUtil;
 import org.apache.poi.ss.usermodel.*;
 import org.apache.poi.xssf.usermodel.XSSFWorkbook;
@@ -182,15 +185,48 @@ public class ExcelValidator {
                 sampleCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
             }
 
-            if (sheetName.toLowerCase().endsWith("assay")) {
-                logger.info("Processing " + sheetName);
-                // Obtain the specific assay first due to potential copy issues
-                ArrayList<Metadata> assaySet = metadataSet.get(sheetName.replaceAll(".*_",""));
-                // Obtain the parent assay
-                assaySet.addAll(metadataSet.get("Assay"));
-                // Parse the assay
-                assayCreation(workbook.getSheet(sheetName), assaySet);
+            if (sheetName.toLowerCase().contains("assay")) {
+                String packageName;
+                if (sheetName.toLowerCase().contains(" - ")) {
+                    packageName = sheetName.split(" - ", 2)[1];
+                } else {
+                    String name = sheetName.replace("Assay", "");
+                    if (name.length() > 0) {
+                        packageName = name.toLowerCase();
+                    } else {
+                        packageName = "core";
+                    }
+                }
+
+                ArrayList<Metadata> metadataArrayList = new ArrayList<>();
+
+                metadataSet.get("Assay").forEach(e -> {
+                    // Add elements from package name
+                    if (e.getPackage().equalsIgnoreCase(packageName)) {
+                        metadataArrayList.add(e);
+                    }
+                    // Add core elements if package name was not core
+                    if (!packageName.equalsIgnoreCase("core") && e.getPackage().equalsIgnoreCase("core")) {
+                        metadataArrayList.add(e);
+                    }
+                });
+
+                // Process the sample sheet
+                assayCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
+                // sampleCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
             }
+            // Removes additional names of the sheet for now
+            // TODO check sample processing
+//            String cleanSheetName = sheetName.replaceAll(".*_", "").replaceAll(".* ", "");
+//            if (cleanSheetName.toLowerCase().endsWith("assay")) {
+//                logger.info("Processing " + cleanSheetName);
+//                // Obtain the specific assay first due to potential copy issues
+//                ArrayList<Metadata> assaySet = metadataSet.get(cleanSheetName);
+//                // Obtain the parent assay
+//                assaySet.addAll(metadataSet.get("Assay"));
+//                // Parse the assay
+//                assayCreation(workbook.getSheet(sheetName), assaySet, null);
+//            }
         }
 
         if (!sampleFound) {
@@ -293,7 +329,7 @@ public class ExcelValidator {
             Data.uploadIrodsFile(new Connection().getIrodsAccount(), xlsxFile, new File(destination + "/" + remotexlsxFile.getName()));
             logging("\n\nUploading database file to the data management system");
             Data.uploadIrodsFile(new Connection().getIrodsAccount(), turtleFile, new File(destination + "/" + remoteturtleFile.getName()));
-            logging("\n\nFiles uploaded to: " + destination);
+            logging("\n\nFiles uploaded to: " + destination + "/");
             logging("\nExcel file: " + xlsxFile);
             logging("\nTurtle file: " + turtleFile);
 
@@ -709,9 +745,10 @@ public class ExcelValidator {
     /**
      * @param sheet
      * @param metadataArrayList
+     * @param packageName
      * @throws Exception
      */
-    private void assayCreation(Sheet sheet, ArrayList<Metadata> metadataArrayList) throws Exception {
+    private void assayCreation(Sheet sheet, ArrayList<Metadata> metadataArrayList, String packageName) throws Exception {
         logger.info("Parsing " + sheet.getSheetName());
         ArrayList<String> header = new ArrayList<>();
 
@@ -723,7 +760,9 @@ public class ExcelValidator {
             assayType = AssayType.AmpliconLibrary;
         } else if (sheet.getSheetName().endsWith("RNAseqAssay")) {
             assayType = AssayType.Transcriptomics;
-        } else if (sheet.getSheetName().endsWith("DNAseqAssay")) {
+        } else if (sheet.getSheetName().toLowerCase().endsWith("illumina")) {
+            assayType = AssayType.Genomics;
+        } else if (sheet.getSheetName().toLowerCase().endsWith("nanopore")) {
             assayType = AssayType.Genomics;
         } else if (sheet.getSheetName().endsWith("ProteomicsAssay")) {
             assayType = AssayType.Proteomics;
@@ -992,7 +1031,6 @@ public class ExcelValidator {
                     }
                     // Parse all other columns
                     otherColumns(header, parsedHeaders, row, assay.getResource().getURI(), metadataArrayList);
-
                     // Coupling the assay to the sample
                     sample.addHasPart(assay);
                 }

src/main/java/nl/fairbydesign/views/template/TemplateView.java

@@ -16,6 +16,7 @@ import com.vaadin.flow.component.icon.VaadinIcon;
 import com.vaadin.flow.component.notification.Notification;
 import com.vaadin.flow.component.orderedlayout.HorizontalLayout;
 import com.vaadin.flow.component.orderedlayout.VerticalLayout;
+import com.vaadin.flow.component.textfield.NumberField;
 import com.vaadin.flow.component.textfield.TextArea;
 import com.vaadin.flow.component.textfield.TextField;
 import com.vaadin.flow.data.binder.Binder;
@@ -115,17 +116,10 @@ public class TemplateView extends Div {
         createPackageSelectionGrid(sampleInformation, "Sample", "help/sample.txt", true);
         accordion.add("Sample Information", sampleInformation);
 
-        // Based on the template create the views per tab in the Excel sheet
-        for (String key : metadataSet.keySet()) {
-            // Only assay subtypes not assay itself
-            if (key.toLowerCase().contains("assay") && !key.toLowerCase().matches("assay")) {
-                logger.debug("Processing element " + key);
-                VerticalLayout assayLayout = new VerticalLayout();
-                createPackageSelectionGrid(assayLayout, key, "help/assay/"+key.toLowerCase()+".txt", false);
-                // createSelectableGrid(assayLayout, key, "help/assay/"+key.toLowerCase()+".txt", false);
-                accordion.add(key + " Information", assayLayout);
-            }
-        }
+        // Assay form with + one more button
+        VerticalLayout assayLayout = new VerticalLayout();
+        createPackageSelectionGrids(assayLayout, "Assay", "help/assay/assay.txt", false);
+        accordion.add("Assay Information", assayLayout);
 
         // Generate and "download" button
         Button buttonGenerate = new Button("GENERATE EXCEL");
@@ -270,6 +264,37 @@ public class TemplateView extends Div {
         // add(buttonDownload);
     }
 
+    private void createPackageSelectionGrids(VerticalLayout assayLayout, String metadata, String help, boolean visible) {
+        // assay layout is the overall placeholder with a button to add more assay layouts
+        ArrayList<VerticalLayout> verticalLayouts = new ArrayList<>();
+        Button addAssayButton = new Button("Add assay");
+        Button removeAssayButton = new Button("Remove assay");
+
+        // Holder for a specific assay
+        VerticalLayout placeholder = new VerticalLayout();
+        createPackageSelectionGrid(placeholder, metadata, help, false);
+        verticalLayouts.add(placeholder);
+        assayLayout.add(addAssayButton, removeAssayButton);
+        assayLayout.add(verticalLayouts.get(0));
+
+        // Button to add another assay to the assay layout
+        addAssayButton.addClickListener(clicked -> {
+                    // Add another assay!
+                    VerticalLayout verticalLayout = new VerticalLayout();
+                    createPackageSelectionGrid(verticalLayout, metadata, help, false);
+                    verticalLayouts.add(verticalLayout);
+                    assayLayout.add(verticalLayout);
+                });
+
+        // Button to remove another assay from the assay layout
+        removeAssayButton.addClickListener(clicked -> {
+            if (verticalLayouts.size() > 0) {
+                VerticalLayout verticalLayout = verticalLayouts.get(verticalLayouts.size() - 1);
+                assayLayout.remove(verticalLayout);
+                verticalLayouts.remove(verticalLayouts.size() - 1);
+            }
+        });
+    }
     /**
      * Creating the package selection interface based on multiple environmental packages according to MIxS
      *
@@ -432,6 +457,8 @@ public class TemplateView extends Div {
             items.get().removeAll(items.get());
             items.get().addAll(uniqueList);
 
+            // Change label of grid based on package including its type
+            gridSelection.setId(metadata + " - " + listener.getValue());
             // Add the items and select the obligatory items
             gridSelection.setItems(items.get());
             // Select obligatory items
@@ -512,6 +539,7 @@ public class TemplateView extends Div {
         HashMap<String, ArrayList<Metadata>> hashMap = new HashMap<>();
         for (Grid grid : gridList) {
             if (grid.isEnabled()) {
+                System.err.println("Grid selection: " + grid.getId().get());
                 logger.info("Grid selection: " + grid.getId().get());
                 String key = grid.getId().get();
                 ArrayList<Metadata> selection = new ArrayList<>();

src/main/java/nl/fairbydesign/views/unlock/export/Export.java

@@ -13,6 +13,7 @@ import com.vaadin.flow.component.html.Label;
 import com.vaadin.flow.component.notification.Notification;
 import com.vaadin.flow.component.orderedlayout.FlexComponent;
 import com.vaadin.flow.component.orderedlayout.HorizontalLayout;
+import com.vaadin.flow.component.orderedlayout.VerticalLayout;
 import com.vaadin.flow.component.textfield.TextField;
 import com.vaadin.flow.data.provider.ListDataProvider;
 import com.vaadin.flow.data.value.ValueChangeMode;
@@ -45,6 +46,8 @@ import java.io.PrintWriter;
 import java.util.ArrayList;
 import java.util.HashSet;
 
+import static com.vaadin.flow.component.orderedlayout.FlexComponent.Alignment.CENTER;
+
 @Route(value = "export", layout = MainView.class)
 @PageTitle("Export module")
 @CssImport("./styles/views/empty/empty-view.css")
@@ -139,8 +142,6 @@ public class Export extends Div {
                 Notification.show("Selected " + grid.getSelectedItems().size());
             });
 
-            add(grid);
-
             Button button = new Button("Generate");
 
             TextField textFieldPrefix = new TextField();
@@ -166,10 +167,16 @@ public class Export extends Div {
                 grid.getDataProvider().refreshAll();
             });
 
-            HorizontalLayout horizontalLayout = new HorizontalLayout();
-            horizontalLayout.add(comboBox, button, textFieldPrefix); // , textFieldReadLength, database, rejectedASVcheckBox);
-            horizontalLayout.setAlignItems(FlexComponent.Alignment.CENTER);
-            add(horizontalLayout);
+            VerticalLayout verticalLayout = new VerticalLayout();
+            verticalLayout.add(comboBox, textFieldPrefix); // , textFieldReadLength, database, rejectedASVcheckBox);
+            verticalLayout.setAlignItems(CENTER);
+            add(verticalLayout);
+
+            add(grid);
+            VerticalLayout verticalLayoutButton = new VerticalLayout();
+            verticalLayoutButton.add(button);
+            verticalLayoutButton.setAlignItems(CENTER);
+            add(verticalLayoutButton);
 
             button.addClickListener(e -> {
                 if (textFieldPrefix.getValue().length() == 0) {
@@ -182,11 +189,10 @@ public class Export extends Div {
                         if (!textFieldPrefix.getValue().matches("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*")) {
                             Notification.show("Only alphanumeric characters and '-' allowed", 10000, Notification.Position.MIDDLE);
                         } else {
-
                             IRODSAccount irodsAccount = credentials.getIrodsAccount();
                             IRODSFileFactory fileFactory = IRODSFileSystem.instance().getIRODSFileFactory(credentials.getIrodsAccount());
 
-                            ArrayList<String> folders = new ArrayList<>();
+                            HashSet<String> folders = new HashSet<>();
 
                             // Obtain all hdt files
                             for (Biom selectedItem : grid.getSelectedItems()) {
@@ -204,7 +210,11 @@ public class Export extends Div {
                                 // Obtain files
                                 pWriter.println(folder);
                                 // Obtain TURTLE files from the investigation
-                                investigations.add(folder.split("/STU_")[0]);
+                                if (folder.contains("/STU_"))
+                                    investigations.add(folder.split("/STU_")[0]);
+                                // Backwards compatibility for S_
+                                if (folder.contains("/S_"))
+                                    investigations.add(folder.split("/S_")[0]);
                             }
 
                             // List and add turtle files

src/test/java/nl/fairbydesign/munlock/ExcelTest.java

@@ -70,7 +70,8 @@ public class ExcelTest {
 
     @Test
     public void testProperty() throws Exception {
-        File excelFile = new File("PROJECT_ID-8.xlsx");
+//        File excelFile = new File("PROJECT_ID-8.xlsx");
+        File excelFile = new File("~/Downloads/mgnify_rhizosphere_selected_addition_modified.xlsx");
         if (excelFile.exists()) {
             XSSFWorkbook workbook = new XSSFWorkbook(excelFile);
             POIXMLProperties props = workbook.getProperties(); // .getCustomProperties().getProperty("sample");
@@ -83,12 +84,13 @@ public class ExcelTest {
 
     @Test
     public void validate() throws Exception {
-        File excelFile = new File("/Users/jasperk/BO3B.xlsx");
+//        File excelFile = new File("/Users/jasperk/BO3B.xlsx");
+        File excelFile = new File("/Users/jasperk/Wageningen University & Research/UNLOCK - FAIR Data Platform Projects - FAIR Data Platform Projects/Projects/UNLOCK/COMP-TAX-FUNC/mgnify_rhizosphere_selected_addition_modified.xlsx");
         if (excelFile.exists()) {
             FileInputStream inputStream = new FileInputStream(excelFile);
             ExcelValidator excelValidator = new ExcelValidator();
             Connection connection = new Connection();
-            excelValidator.validate("BO3B", inputStream, connection.getIrodsAccount());
+            excelValidator.validate("mgnify_rhizosphere_selected_addition_modified", inputStream, connection.getIrodsAccount());
         } else {
             logger.warn("Test file does not exists");
         }