Skip to content

Bug #13230: upgrade CSV metadata #77

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
laedanrex wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Bug #13230: upgrade CSV metadata #77

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 33 additions & 85 deletions sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/CSVMetadataFormatter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
package fr.gouv.vitam.tools.sedalib.inout.importer;

import fr.gouv.vitam.tools.sedalib.inout.importer.model.MetadataTag;
import fr.gouv.vitam.tools.sedalib.inout.importer.model.ValueAttrMetadataTag;
import fr.gouv.vitam.tools.sedalib.utils.SEDALibException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.text.StringEscapeUtils;

import java.nio.file.Path;
Expand All @@ -16,60 +19,7 @@
* It's a utility class for {@link CSVMetadataToDataObjectPackageImporter} to analyse header line, and then interpret other lines through the formatter.
*/
public class CSVMetadataFormatter {
/**
* Utility class for metadata formatter.
*/
private class MetadataTag {

String name;
String value;
String attr;
MetadataTag parent;
LinkedHashMap<Integer, List<MetadataTag>> subTags;

/**
* Instantiates a new Metadata tag.
*
* @param name the name
* @param parent the parent
*/
protected MetadataTag(String name, MetadataTag parent) throws SEDALibException {
if ((name != null) && (!name.matches("[a-zA-Z0-9_-]+"))) {
throw new SEDALibException("Caractère interdit dans le tag XML [" + name + "]");
}
this.name = name;
this.value = null;
this.attr = null;
this.subTags = null;
this.parent = parent;
}

@Override
public String toString() {
if (parent == null) {
return name;
} else {
return parent.toString() + "." + name;
}
}
}


private class ValueAttrMetadataTag {
boolean isValue;
MetadataTag tag;

/**
* Instantiates a new Value attr metadata tag.
*
* @param isValue the is value
* @param tag the tag
*/
public ValueAttrMetadataTag(boolean isValue, MetadataTag tag) {
this.isValue = isValue;
this.tag = tag;
}
}

private static final String ID = "id";
private static final String FILE = "file";
Expand All @@ -83,7 +33,7 @@ public ValueAttrMetadataTag(boolean isValue, MetadataTag tag) {
ID, FILE, PARENTID, PARENTFILE, OBJECTFILES
);
private MetadataTag rootTag, contentTag, managementTag;
private LinkedHashMap<Integer, ValueAttrMetadataTag> tagHeaderColumnMapping;
private List<ValueAttrMetadataTag> tagHeaderColumnMapping;
private int numberOfMandatoryHeaderFound;
private int columnCount;

Expand All @@ -93,7 +43,7 @@ public ValueAttrMetadataTag(boolean isValue, MetadataTag tag) {
private int fileColumn;
private int objectfilesColumn;
private int parentGUIDColumn;
private boolean isOnlyFile;
private boolean isOnlyFile = false;


private void analyseFirstColumns(String[] headerRow) throws SEDALibException {
Expand All @@ -113,31 +63,26 @@ private void analyseFirstColumns(String[] headerRow) throws SEDALibException {
objectfilesColumn = -1;
parentGUIDColumn = -1;
} else if (numberOfMandatoryHeaderFound == 2 && firstsMandatoryHeadersFound.containsAll(List.of(FILE, PARENTFILE))) {
isOnlyFile = false;
guidColumn = firstsMandatoryHeadersFound.indexOf(FILE);
fileColumn = guidColumn;
objectfilesColumn = -1;
parentGUIDColumn = firstsMandatoryHeadersFound.indexOf(PARENTFILE);
} else if (numberOfMandatoryHeaderFound == 3 && firstsMandatoryHeadersFound.containsAll(List.of(FILE, PARENTFILE, ID))) {
isOnlyFile = false;
guidColumn = firstsMandatoryHeadersFound.indexOf(ID);
fileColumn = firstsMandatoryHeadersFound.indexOf(FILE);
objectfilesColumn = -1;
parentGUIDColumn = firstsMandatoryHeadersFound.indexOf(PARENTFILE);
} else if (numberOfMandatoryHeaderFound == 3 && firstsMandatoryHeadersFound.containsAll(List.of(FILE, PARENTID, ID))) {
isOnlyFile = false;
guidColumn = firstsMandatoryHeadersFound.indexOf(ID);
fileColumn = firstsMandatoryHeadersFound.indexOf(FILE);
objectfilesColumn = -1;
parentGUIDColumn = firstsMandatoryHeadersFound.indexOf(PARENTID);
} else if (numberOfMandatoryHeaderFound == 3 && firstsMandatoryHeadersFound.containsAll(List.of(OBJECTFILES, PARENTID, ID))) {
isOnlyFile = false;
guidColumn = firstsMandatoryHeadersFound.indexOf(ID);
fileColumn = -1;
objectfilesColumn = firstsMandatoryHeadersFound.indexOf(OBJECTFILES);
parentGUIDColumn = firstsMandatoryHeadersFound.indexOf(PARENTID);
} else if (numberOfMandatoryHeaderFound == 4 && firstsMandatoryHeadersFound.containsAll(List.of(FILE, OBJECTFILES, PARENTID, ID))) {
isOnlyFile = false;
guidColumn = firstsMandatoryHeadersFound.indexOf(ID);
objectfilesColumn = firstsMandatoryHeadersFound.indexOf(OBJECTFILES);
fileColumn = firstsMandatoryHeadersFound.indexOf(FILE);
Expand Down Expand Up @@ -208,7 +153,7 @@ private MetadataTag getTag(MetadataTag tag, List<String> splittedMetadataName) t

private void analyseTags(String[] headerRow) throws SEDALibException {
MetadataTag currentTag = null;
ValueAttrMetadataTag vamt;
ValueAttrMetadataTag valueAttrMetadataTag;

if (headerRow.length <= numberOfMandatoryHeaderFound) {
throw new SEDALibException("Pas de colonne de métadonnées.");
Expand All @@ -221,21 +166,23 @@ private void analyseTags(String[] headerRow) throws SEDALibException {
contentTag = rootTag;
}
managementTag = null;
tagHeaderColumnMapping = new LinkedHashMap<>();
tagHeaderColumnMapping = new ArrayList<>();
for (int i = numberOfMandatoryHeaderFound; i < headerRow.length; i++) {
if (headerRow[i].equalsIgnoreCase("attr")) {
String value = headerRow[i];
if (value.equalsIgnoreCase("attr")) {
if (currentTag == null) {
throw new SEDALibException("Le header attr en colonne n°" + i + " ne peut pas s'appliquer.");
}
vamt = new ValueAttrMetadataTag(false, currentTag);
} else if (headerRow[i].endsWith(".attr")) {
currentTag = getTag(rootTag, new ArrayList<>(Arrays.asList(headerRow[i].split("\\."))));
vamt = new ValueAttrMetadataTag(false, currentTag);
valueAttrMetadataTag = new ValueAttrMetadataTag(false, currentTag);
} else if (value.endsWith(".attr")) {
currentTag = getTag(rootTag, new ArrayList<>(Arrays.asList(value.split("\\."))));
valueAttrMetadataTag = new ValueAttrMetadataTag(false, currentTag);
} else {
currentTag = getTag(rootTag, new ArrayList<>(Arrays.asList(headerRow[i].split("\\."))));
vamt = new ValueAttrMetadataTag(true, currentTag);
currentTag = getTag(rootTag, new ArrayList<>(Arrays.asList(value.split("\\."))));

valueAttrMetadataTag = new ValueAttrMetadataTag(true, currentTag);
}
tagHeaderColumnMapping.put(i, vamt);
tagHeaderColumnMapping.add(valueAttrMetadataTag);
}
if (contentTag == null) {
throw new SEDALibException("Pas de colonne de métadonnées Content.");
Expand All @@ -257,14 +204,14 @@ public CSVMetadataFormatter(String[] headerRow, Path baseDir) throws SEDALibExce
}

private void resetValues() {
tagHeaderColumnMapping.values().stream().forEach(valueAttr -> {
tagHeaderColumnMapping.forEach(valueAttr -> {
valueAttr.tag.value = null;
valueAttr.tag.attr = null;
});
}

private void defineColumnValue(int headerColumn, String cell) {
ValueAttrMetadataTag vamt = tagHeaderColumnMapping.get(headerColumn);
ValueAttrMetadataTag vamt = tagHeaderColumnMapping.get(headerColumn - 1);
if (vamt.isValue) {
vamt.tag.value = cell;
} else {
Expand All @@ -286,46 +233,47 @@ private String getOneSubTagXML(MetadataTag tag, String subTagName) {
}

private boolean notEmptyValues(MetadataTag tag) {
boolean result = false;
if (tag.subTags == null) {
return !((tag.value == null) || tag.value.isEmpty());
return StringUtils.isNotEmpty(tag.value);
}
for (List<MetadataTag> tagList : tag.subTags.values()) {
for (MetadataTag subTag : tagList) {
result = result || notEmptyValues(subTag);
if (notEmptyValues(subTag)) {
return true;
}
}
}
return result;
return false;
}

private String generateRuleTypeTagXML(MetadataTag tag) throws SEDALibException {
String result = "";
StringBuilder result = new StringBuilder();
for (List<MetadataTag> tagList : tag.subTags.values()) {
for (MetadataTag mt : tagList) {
if (mt.name.equals("Rule") && !mt.value.isEmpty()) {
result += "<Rule>" + StringEscapeUtils.escapeXml10(mt.value) + "</Rule>";
result.append("<Rule>").append(StringEscapeUtils.escapeXml10(mt.value)).append("</Rule>");
mt.value = null;
}
}
for (MetadataTag mt : tagList) {
if (mt.name.equals("StartDate") && !mt.value.isEmpty()) {
result += "<StartDate>" + StringEscapeUtils.escapeXml10(mt.value) + "</StartDate>";
result.append("<StartDate>").append(StringEscapeUtils.escapeXml10(mt.value)).append("</StartDate>");
mt.value = null;
}
}
}
result += getOneSubTagXML(tag, "PreventInheritance");
result += getOneSubTagXML(tag, "RefNonRuleId");
result += getOneSubTagXML(tag, "FinalAction");
result.append(getOneSubTagXML(tag, "PreventInheritance"));
result.append(getOneSubTagXML(tag, "RefNonRuleId"));
result.append(getOneSubTagXML(tag, "FinalAction"));

if (notEmptyValues(tag)) {
throw new SEDALibException("La règle [" + tag.name + "] contient des champs non conformes SEDA.");
}

if (!result.isEmpty()) {
result = "<" + tag.name + ">" + result + "</" + tag.name + ">";
if (result.length() > 0) {
result = new StringBuilder("<" + tag.name + ">" + result + "</" + tag.name + ">");
}
return result;
return result.toString();
}

private String generateHoldRuleTagXML(MetadataTag tag) throws SEDALibException {
Expand Down
63 changes: 7 additions & 56 deletions ...va/fr/gouv/vitam/tools/sedalib/inout/importer/CSVMetadataToDataObjectPackageImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@
import fr.gouv.vitam.tools.sedalib.core.BinaryDataObject;
import fr.gouv.vitam.tools.sedalib.core.DataObjectGroup;
import fr.gouv.vitam.tools.sedalib.core.DataObjectPackage;
import fr.gouv.vitam.tools.sedalib.inout.importer.model.Line;
import fr.gouv.vitam.tools.sedalib.utils.SEDALibException;
import fr.gouv.vitam.tools.sedalib.utils.SEDALibProgressLogger;

Expand Down Expand Up @@ -94,60 +95,6 @@
*/
public class CSVMetadataToDataObjectPackageImporter {

private class Line {
/**
* The Guid.
*/
String guid;
/**
* The Parent guid.
*/
String parentGUID;
/**
* The File.
*/
String file;

/**
* The Object Files.
*/
List<String> objectFiles;

/**
* The Content xml metadata.
*/
String contentXMLMetadata;
/**
* The Management xml metadata.
*/
String managementXMLMetadata;
/**
* The Au.
*/
ArchiveUnit au;

/**
* Instantiates a new Line.
*
* @param guid the guid
* @param parentGUID the parent guid
* @param file the file
* @param contentXMLMetadata the content xml metadata
*/
public Line(String guid, String parentGUID, String file, String objectFiles, String contentXMLMetadata, String managementXMLMetadata) {
this.guid = guid;
this.parentGUID = parentGUID;
this.file = file;
if (objectFiles.trim().isEmpty())
this.objectFiles = Arrays.asList();
else
this.objectFiles = Arrays.asList(objectFiles.split("\\|"));
this.contentXMLMetadata = contentXMLMetadata;
this.managementXMLMetadata = managementXMLMetadata;
this.au = null;
}
}

/**
* The csv metadata file name .
*/
Expand Down Expand Up @@ -241,8 +188,12 @@ private boolean readCSVFile() throws SEDALibException, InterruptedException {
continue;
}
try {
currentLine = new Line(metadataFormatter.getGUID(row), metadataFormatter.getParentGUID(row), //NOSONAR
metadataFormatter.getFile(row), metadataFormatter.getObjectFiles(row), metadataFormatter.doFormatAndExtractContentXML(row), metadataFormatter.extractManagementXML());
currentLine = new Line(metadataFormatter.getGUID(row),
metadataFormatter.getParentGUID(row), //NOSONAR
metadataFormatter.getFile(row),
metadataFormatter.getObjectFiles(row),
metadataFormatter.doFormatAndExtractContentXML(row),
metadataFormatter.extractManagementXML());
} catch (SEDALibException e) {
throw new SEDALibException("Erreur sur la ligne "+lineCount, e);
}
Expand Down
39 changes: 39 additions & 0 deletions sedalib/src/main/java/fr/gouv/vitam/tools/sedalib/inout/importer/model/Line.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package fr.gouv.vitam.tools.sedalib.inout.importer.model;


import fr.gouv.vitam.tools.sedalib.core.ArchiveUnit;

import java.util.Arrays;
import java.util.Collections;
import java.util.List;

public class Line {

public String guid;
public String parentGUID;
public String file;
public List<String> objectFiles;

/**
* everything inside <content></content>
*/
public String contentXMLMetadata;
/**
* everything inside <management></management>
*/
public String managementXMLMetadata;
public ArchiveUnit au;

public Line(String guid, String parentGUID, String file, String objectFiles, String contentXMLMetadata, String managementXMLMetadata) {
this.guid = guid;
this.parentGUID = parentGUID;
this.file = file;
if (objectFiles.trim().isEmpty())
this.objectFiles = Collections.emptyList();
else
this.objectFiles = Arrays.asList(objectFiles.split("\\|"));
this.contentXMLMetadata = contentXMLMetadata;
this.managementXMLMetadata = managementXMLMetadata;
this.au = null;
}
}
Loading