Commit 9b356217 authored by Jasper Koehorst's avatar Jasper Koehorst
Browse files

Sync interface, assay now incremental for adding other packages, name in sheetname and export fixes

parent af8a0085
Pipeline #36541 passed with stage
in 3 minutes and 46 seconds
This diff is collapsed.
......@@ -68,7 +68,6 @@
"lit-element": "2.5.1",
"lit-html": "1.4.1",
"terser": "4.6.7"
},
"devDependencies": {
"babel-loader": "8.2.2",
......@@ -178,6 +177,6 @@
"webpack-dev-server": "3.11.0",
"webpack-merge": "4.2.2"
},
"hash": "c026424a89aa150aada71c0fce2135327cdd92025da4c8a55e6453aff50825d0"
"hash": "8587df101e8898757cb19c3eb0932ffc91999721c9c0def3702d8a367cdd6e1f"
}
}
......@@ -138,7 +138,7 @@ public class MetadataParser extends Metadata {
// email value from https://www.emailregex.com
syntax = syntax.replace("{email}", "(?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|\"(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21\\x23-\\x5b\\x5d-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])*\")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\\[(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?|[a-z0-9-]*[a-z0-9]:(?:[\\x01-\\x08\\x0b\\x0c\\x0e-\\x1f\\x21-\\x5a\\x53-\\x7f]|\\\\[\\x01-\\x09\\x0b\\x0c\\x0e-\\x7f])+)\\])");
// orcid regex
syntax = syntax.replace("{orcid}", "(\\d|X{4}-){3}\\d|X{4}$");
syntax = syntax.replace("{orcid}", "^(\\d|X{4}-){3}\\d|X{4}$");
// Day(s) unit
syntax = syntax.replace("{day}", "day[s]?");
// Units are to broad so it becomes free text
......@@ -284,7 +284,7 @@ public class MetadataParser extends Metadata {
SortedMap<String, ArrayList<Metadata>> sheetObjects = new TreeMap<>();
for (int i = 0; i < workbook.getNumberOfSheets(); i++) {
Sheet sheet = workbook.getSheetAt(i);
logger.debug("Processing " + sheet.getSheetName() + " " + sheet.getLastRowNum());
logger.info("Processing " + sheet.getSheetName() + " " + sheet.getLastRowNum());
String name = sheet.getSheetName();
ArrayList<Metadata> metadataHashSet = MetadataParser.parser(sheet);
sheetObjects.put(name, metadataHashSet);
......
......@@ -44,9 +44,9 @@ public class ExcelGenerator {
workbook = new XSSFWorkbook();
// set sample package property
POIXMLProperties props = workbook.getProperties();
POIXMLProperties.CustomProperties custProp = props.getCustomProperties();
custProp.addProperty("sample", samplePackage);
// POIXMLProperties props = workbook.getProperties();
// POIXMLProperties.CustomProperties custProp = props.getCustomProperties();
//custProp.addProperty("sample", samplePackage);
// Create excel style object
makeHeaderCellStyle();
......@@ -72,21 +72,28 @@ public class ExcelGenerator {
for (String key : gridSelection.keySet()) {
if (key.toLowerCase().contains("sample")) {
// Overwrite key name by including package name
String sheetName = key + " - " + samplePackage;
generateAssaySheet(workbook, sheetName, gridSelection.get(key));
generateAssaySheet(workbook, key, gridSelection.get(key));
remove.add(key);
}
}
// Create assays, all for loops are separated to preserve order
for (String key : gridSelection.keySet()) {
if (key.toLowerCase().contains("assay")) {
// Overwrite key name by including package name
generateAssaySheet(workbook, key, gridSelection.get(key));
remove.add(key);
break;
}
}
// Removing the observation unit and sample object
gridSelection.keySet().removeAll(remove);
// gridSelection.keySet().removeAll(remove);
// Perform the rest of the assays
for (String key : gridSelection.keySet()) {
logger.info("Creating assay sheet " + key);
generateAssaySheet(workbook, key, gridSelection.get(key));
}
// for (String key : gridSelection.keySet()) {
// logger.info("Creating assay sheet " + key);
// generateAssaySheet(workbook, key, gridSelection.get(key));
// }
// Save file and escape the / in a name
File filename = new File(project.getIdentifier().replaceAll("/","-") + ".xlsx");
......
......@@ -11,6 +11,9 @@ import nl.wur.ssb.RDFSimpleCon.api.Domain;
import org.apache.commons.codec.digest.DigestUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.jena.rdf.model.NodeIterator;
import org.apache.jena.rdf.model.RDFList;
import org.apache.jena.rdf.model.Resource;
import org.apache.jena.rdf.model.StmtIterator;
import org.apache.poi.hssf.usermodel.HSSFDateUtil;
import org.apache.poi.ss.usermodel.*;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
......@@ -182,15 +185,48 @@ public class ExcelValidator {
sampleCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
}
if (sheetName.toLowerCase().endsWith("assay")) {
logger.info("Processing " + sheetName);
// Obtain the specific assay first due to potential copy issues
ArrayList<Metadata> assaySet = metadataSet.get(sheetName.replaceAll(".*_",""));
// Obtain the parent assay
assaySet.addAll(metadataSet.get("Assay"));
// Parse the assay
assayCreation(workbook.getSheet(sheetName), assaySet);
if (sheetName.toLowerCase().contains("assay")) {
String packageName;
if (sheetName.toLowerCase().contains(" - ")) {
packageName = sheetName.split(" - ", 2)[1];
} else {
String name = sheetName.replace("Assay", "");
if (name.length() > 0) {
packageName = name.toLowerCase();
} else {
packageName = "core";
}
}
ArrayList<Metadata> metadataArrayList = new ArrayList<>();
metadataSet.get("Assay").forEach(e -> {
// Add elements from package name
if (e.getPackage().equalsIgnoreCase(packageName)) {
metadataArrayList.add(e);
}
// Add core elements if package name was not core
if (!packageName.equalsIgnoreCase("core") && e.getPackage().equalsIgnoreCase("core")) {
metadataArrayList.add(e);
}
});
// Process the sample sheet
assayCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
// sampleCreation(workbook.getSheet(sheetName), metadataArrayList, packageName);
}
// Removes additional names of the sheet for now
// TODO check sample processing
// String cleanSheetName = sheetName.replaceAll(".*_", "").replaceAll(".* ", "");
// if (cleanSheetName.toLowerCase().endsWith("assay")) {
// logger.info("Processing " + cleanSheetName);
// // Obtain the specific assay first due to potential copy issues
// ArrayList<Metadata> assaySet = metadataSet.get(cleanSheetName);
// // Obtain the parent assay
// assaySet.addAll(metadataSet.get("Assay"));
// // Parse the assay
// assayCreation(workbook.getSheet(sheetName), assaySet, null);
// }
}
if (!sampleFound) {
......@@ -293,7 +329,7 @@ public class ExcelValidator {
Data.uploadIrodsFile(new Connection().getIrodsAccount(), xlsxFile, new File(destination + "/" + remotexlsxFile.getName()));
logging("\n\nUploading database file to the data management system");
Data.uploadIrodsFile(new Connection().getIrodsAccount(), turtleFile, new File(destination + "/" + remoteturtleFile.getName()));
logging("\n\nFiles uploaded to: " + destination);
logging("\n\nFiles uploaded to: " + destination + "/");
logging("\nExcel file: " + xlsxFile);
logging("\nTurtle file: " + turtleFile);
......@@ -709,9 +745,10 @@ public class ExcelValidator {
/**
* @param sheet
* @param metadataArrayList
* @param packageName
* @throws Exception
*/
private void assayCreation(Sheet sheet, ArrayList<Metadata> metadataArrayList) throws Exception {
private void assayCreation(Sheet sheet, ArrayList<Metadata> metadataArrayList, String packageName) throws Exception {
logger.info("Parsing " + sheet.getSheetName());
ArrayList<String> header = new ArrayList<>();
......@@ -723,7 +760,9 @@ public class ExcelValidator {
assayType = AssayType.AmpliconLibrary;
} else if (sheet.getSheetName().endsWith("RNAseqAssay")) {
assayType = AssayType.Transcriptomics;
} else if (sheet.getSheetName().endsWith("DNAseqAssay")) {
} else if (sheet.getSheetName().toLowerCase().endsWith("illumina")) {
assayType = AssayType.Genomics;
} else if (sheet.getSheetName().toLowerCase().endsWith("nanopore")) {
assayType = AssayType.Genomics;
} else if (sheet.getSheetName().endsWith("ProteomicsAssay")) {
assayType = AssayType.Proteomics;
......@@ -992,7 +1031,6 @@ public class ExcelValidator {
}
// Parse all other columns
otherColumns(header, parsedHeaders, row, assay.getResource().getURI(), metadataArrayList);
// Coupling the assay to the sample
sample.addHasPart(assay);
}
......
......@@ -16,6 +16,7 @@ import com.vaadin.flow.component.icon.VaadinIcon;
import com.vaadin.flow.component.notification.Notification;
import com.vaadin.flow.component.orderedlayout.HorizontalLayout;
import com.vaadin.flow.component.orderedlayout.VerticalLayout;
import com.vaadin.flow.component.textfield.NumberField;
import com.vaadin.flow.component.textfield.TextArea;
import com.vaadin.flow.component.textfield.TextField;
import com.vaadin.flow.data.binder.Binder;
......@@ -115,17 +116,10 @@ public class TemplateView extends Div {
createPackageSelectionGrid(sampleInformation, "Sample", "help/sample.txt", true);
accordion.add("Sample Information", sampleInformation);
// Based on the template create the views per tab in the Excel sheet
for (String key : metadataSet.keySet()) {
// Only assay subtypes not assay itself
if (key.toLowerCase().contains("assay") && !key.toLowerCase().matches("assay")) {
logger.debug("Processing element " + key);
VerticalLayout assayLayout = new VerticalLayout();
createPackageSelectionGrid(assayLayout, key, "help/assay/"+key.toLowerCase()+".txt", false);
// createSelectableGrid(assayLayout, key, "help/assay/"+key.toLowerCase()+".txt", false);
accordion.add(key + " Information", assayLayout);
}
}
// Assay form with + one more button
VerticalLayout assayLayout = new VerticalLayout();
createPackageSelectionGrids(assayLayout, "Assay", "help/assay/assay.txt", false);
accordion.add("Assay Information", assayLayout);
// Generate and "download" button
Button buttonGenerate = new Button("GENERATE EXCEL");
......@@ -270,6 +264,37 @@ public class TemplateView extends Div {
// add(buttonDownload);
}
private void createPackageSelectionGrids(VerticalLayout assayLayout, String metadata, String help, boolean visible) {
// assay layout is the overall placeholder with a button to add more assay layouts
ArrayList<VerticalLayout> verticalLayouts = new ArrayList<>();
Button addAssayButton = new Button("Add assay");
Button removeAssayButton = new Button("Remove assay");
// Holder for a specific assay
VerticalLayout placeholder = new VerticalLayout();
createPackageSelectionGrid(placeholder, metadata, help, false);
verticalLayouts.add(placeholder);
assayLayout.add(addAssayButton, removeAssayButton);
assayLayout.add(verticalLayouts.get(0));
// Button to add another assay to the assay layout
addAssayButton.addClickListener(clicked -> {
// Add another assay!
VerticalLayout verticalLayout = new VerticalLayout();
createPackageSelectionGrid(verticalLayout, metadata, help, false);
verticalLayouts.add(verticalLayout);
assayLayout.add(verticalLayout);
});
// Button to remove another assay from the assay layout
removeAssayButton.addClickListener(clicked -> {
if (verticalLayouts.size() > 0) {
VerticalLayout verticalLayout = verticalLayouts.get(verticalLayouts.size() - 1);
assayLayout.remove(verticalLayout);
verticalLayouts.remove(verticalLayouts.size() - 1);
}
});
}
/**
* Creating the package selection interface based on multiple environmental packages according to MIxS
*
......@@ -432,6 +457,8 @@ public class TemplateView extends Div {
items.get().removeAll(items.get());
items.get().addAll(uniqueList);
// Change label of grid based on package including its type
gridSelection.setId(metadata + " - " + listener.getValue());
// Add the items and select the obligatory items
gridSelection.setItems(items.get());
// Select obligatory items
......@@ -512,6 +539,7 @@ public class TemplateView extends Div {
HashMap<String, ArrayList<Metadata>> hashMap = new HashMap<>();
for (Grid grid : gridList) {
if (grid.isEnabled()) {
System.err.println("Grid selection: " + grid.getId().get());
logger.info("Grid selection: " + grid.getId().get());
String key = grid.getId().get();
ArrayList<Metadata> selection = new ArrayList<>();
......
......@@ -13,6 +13,7 @@ import com.vaadin.flow.component.html.Label;
import com.vaadin.flow.component.notification.Notification;
import com.vaadin.flow.component.orderedlayout.FlexComponent;
import com.vaadin.flow.component.orderedlayout.HorizontalLayout;
import com.vaadin.flow.component.orderedlayout.VerticalLayout;
import com.vaadin.flow.component.textfield.TextField;
import com.vaadin.flow.data.provider.ListDataProvider;
import com.vaadin.flow.data.value.ValueChangeMode;
......@@ -45,6 +46,8 @@ import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.HashSet;
import static com.vaadin.flow.component.orderedlayout.FlexComponent.Alignment.CENTER;
@Route(value = "export", layout = MainView.class)
@PageTitle("Export module")
@CssImport("./styles/views/empty/empty-view.css")
......@@ -139,8 +142,6 @@ public class Export extends Div {
Notification.show("Selected " + grid.getSelectedItems().size());
});
add(grid);
Button button = new Button("Generate");
TextField textFieldPrefix = new TextField();
......@@ -166,10 +167,16 @@ public class Export extends Div {
grid.getDataProvider().refreshAll();
});
HorizontalLayout horizontalLayout = new HorizontalLayout();
horizontalLayout.add(comboBox, button, textFieldPrefix); // , textFieldReadLength, database, rejectedASVcheckBox);
horizontalLayout.setAlignItems(FlexComponent.Alignment.CENTER);
add(horizontalLayout);
VerticalLayout verticalLayout = new VerticalLayout();
verticalLayout.add(comboBox, textFieldPrefix); // , textFieldReadLength, database, rejectedASVcheckBox);
verticalLayout.setAlignItems(CENTER);
add(verticalLayout);
add(grid);
VerticalLayout verticalLayoutButton = new VerticalLayout();
verticalLayoutButton.add(button);
verticalLayoutButton.setAlignItems(CENTER);
add(verticalLayoutButton);
button.addClickListener(e -> {
if (textFieldPrefix.getValue().length() == 0) {
......@@ -182,11 +189,10 @@ public class Export extends Div {
if (!textFieldPrefix.getValue().matches("[a-z0-9]([-a-z0-9]*[a-z0-9])?(\\\\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*")) {
Notification.show("Only alphanumeric characters and '-' allowed", 10000, Notification.Position.MIDDLE);
} else {
IRODSAccount irodsAccount = credentials.getIrodsAccount();
IRODSFileFactory fileFactory = IRODSFileSystem.instance().getIRODSFileFactory(credentials.getIrodsAccount());
ArrayList<String> folders = new ArrayList<>();
HashSet<String> folders = new HashSet<>();
// Obtain all hdt files
for (Biom selectedItem : grid.getSelectedItems()) {
......@@ -204,7 +210,11 @@ public class Export extends Div {
// Obtain files
pWriter.println(folder);
// Obtain TURTLE files from the investigation
investigations.add(folder.split("/STU_")[0]);
if (folder.contains("/STU_"))
investigations.add(folder.split("/STU_")[0]);
// Backwards compatibility for S_
if (folder.contains("/S_"))
investigations.add(folder.split("/S_")[0]);
}
// List and add turtle files
......
......@@ -70,7 +70,8 @@ public class ExcelTest {
@Test
public void testProperty() throws Exception {
File excelFile = new File("PROJECT_ID-8.xlsx");
// File excelFile = new File("PROJECT_ID-8.xlsx");
File excelFile = new File("~/Downloads/mgnify_rhizosphere_selected_addition_modified.xlsx");
if (excelFile.exists()) {
XSSFWorkbook workbook = new XSSFWorkbook(excelFile);
POIXMLProperties props = workbook.getProperties(); // .getCustomProperties().getProperty("sample");
......@@ -83,12 +84,13 @@ public class ExcelTest {
@Test
public void validate() throws Exception {
File excelFile = new File("/Users/jasperk/BO3B.xlsx");
// File excelFile = new File("/Users/jasperk/BO3B.xlsx");
File excelFile = new File("/Users/jasperk/Wageningen University & Research/UNLOCK - FAIR Data Platform Projects - FAIR Data Platform Projects/Projects/UNLOCK/COMP-TAX-FUNC/mgnify_rhizosphere_selected_addition_modified.xlsx");
if (excelFile.exists()) {
FileInputStream inputStream = new FileInputStream(excelFile);
ExcelValidator excelValidator = new ExcelValidator();
Connection connection = new Connection();
excelValidator.validate("BO3B", inputStream, connection.getIrodsAccount());
excelValidator.validate("mgnify_rhizosphere_selected_addition_modified", inputStream, connection.getIrodsAccount());
} else {
logger.warn("Test file does not exists");
}
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment