Commit bdb1bc04 authored by Vincent Calame's avatar Vincent Calame

Mécanisme de sauvegarde d'une copie valide

parent 97420200
......@@ -31,14 +31,13 @@ public class XmlChecker {
this.factory = SAXParserFactory.newInstance();
}
public boolean check(InputStream inputStream) throws SAXException, IOException {
public void check(InputStream inputStream) throws SAXException, IOException {
try {
SAXParser saxParser = factory.newSAXParser();
saxParser.parse(inputStream, new DefaultHandler());
} catch (ParserConfigurationException pce) {
throw new NestedLibraryException(pce);
}
return true;
}
}
/* SctServer_API - Copyright (c) 2013-2017 Vincent Calame - Exemole
/* SctServer_API - Copyright (c) 2013-2018 Vincent Calame - Exemole
* Logiciel libre donné sous triple licence :
* 1) selon les termes de la CeCILL V2
* 2) selon les termes de l’EUPL V.1.1
......@@ -20,7 +20,7 @@ import net.mapeadores.util.primitives.FuzzyDate;
*/
public interface ScrutariInfoContent {
public FuzzyDate getLastUpdate();
public FuzzyDate getDate();
public List<URL> getDataUrlList();
......
/* SctServer - Copyright (c) 2008-2017 Vincent Calame - Exemole
/* SctServer - Copyright (c) 2008-2018 Vincent Calame - Exemole
* Logiciel libre donné sous triple licence :
* 1) selon les termes de la CeCILL V2
* 2) selon les termes de l’EUPL V.1.1
......@@ -11,6 +11,7 @@ package fr.exemole.sctserver.api.storage;
import fr.exemole.sctserver.api.conf.ConfDirectory;
import fr.exemole.sctserver.api.feed.SctFeedCache;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.List;
......@@ -34,7 +35,7 @@ public interface EngineStorage {
public final static String STATUS_NAME = "status";
public List<Data> getDataList(String scrutariSourceName);
public List<StoredData> getDataList(String scrutariSourceName);
public boolean hasData(String scrutariSourceName, URL url);
......@@ -100,18 +101,26 @@ public interface EngineStorage {
}
public interface Data {
public interface MatchedData {
public InputStream getInputStream();
public InputStream getInputStream() throws IOException;
public URL getMatchingURL();
public boolean isNotChanged();
}
public interface MatchedData extends Data {
public interface StoredData {
public URL getMatchingURL();
public InputStream getInputStream() throws IOException;
public boolean isNotChanged();
public boolean hasBackup();
public boolean restoreBackup() throws IOException;
public void backup() throws IOException;
}
......
......@@ -261,31 +261,18 @@ public final class SctEngineImpl implements SctEngine {
private AliasManager checkBaseURI(UpdateLogHandler updateLogHandler, TimeLog timeLog) {
List<ScrutariSource> scrutariSourceList = scrutariSourceManager.getScrutariSourceList();
AliasManager.Builder aliasManagerBuilder = buildFromSource(scrutariSourceList, updateLogHandler);
XmlChecker xmlChecker = new XmlChecker();
BaseURIXmlEngine engine = new BaseURIXmlEngine(updateLogHandler);
for (ScrutariSource scrutariSource : scrutariSourceList) {
String scrutariSourceName = scrutariSource.getScrutariSourceDef().getName();
updateLogHandler.setCurrentScrutariSource(scrutariSourceName);
timeLog.addSection(scrutariSourceName);
List<EngineStorage.Data> dataList = engineStorage.getDataList(scrutariSourceName);
List<EngineStorage.StoredData> dataList = engineStorage.getDataList(scrutariSourceName);
if (dataList.isEmpty()) {
continue;
}
boolean ok = true;
for (EngineStorage.Data data : dataList) {
try (InputStream is = data.getInputStream()) {
boolean stepDone = xmlChecker.check(is);
if (!stepDone) {
ok = false;
}
} catch (SAXException | IOException e) {
scrutariSourceManager.setState(scrutariSource, CollectResult.DATA_NOTXML_STATE);
updateLogHandler.exception(e);
ok = false;
}
}
boolean ok = checkXml(scrutariSource, dataList, updateLogHandler);
if (ok) {
for (EngineStorage.Data data : dataList) {
for (EngineStorage.StoredData data : dataList) {
try (InputStream is = data.getInputStream()) {
BaseURI baseURI = engine.read(is);
if (baseURI != null) {
......@@ -311,10 +298,45 @@ public final class SctEngineImpl implements SctEngine {
return aliasManagerBuilder.toAliasManager();
}
private boolean checkXml(ScrutariSource scrutariSource, List<EngineStorage.StoredData> dataList, UpdateLogHandler updateLogHandler) {
XmlChecker xmlChecker = new XmlChecker();
boolean ok = true;
for (EngineStorage.StoredData data : dataList) {
boolean done = false;
try (InputStream is = data.getInputStream()) {
xmlChecker.check(is);
done = true;
} catch (SAXException | IOException e) {
scrutariSourceManager.setState(scrutariSource, CollectResult.DATA_NOTXML_STATE);
updateLogHandler.exception(e);
}
if ((!done) && (data.hasBackup())) {
boolean restored = false;
try {
restored = data.restoreBackup();
} catch (IOException ioe) {
updateLogHandler.exception(ioe);
}
if (restored) {
try (InputStream is = data.getInputStream()) {
xmlChecker.check(is);
done = true;
} catch (SAXException | IOException e) {
updateLogHandler.exception(e);
}
}
}
if (!done) {
ok = false;
}
}
return ok;
}
private void parseXml(DataCacheWriter dataCacheWriter, DataCoder dataCoder, AliasManager aliasManager, UpdateLogHandler updateLogHandler, TimeLog timeLog) {
BaseXmlEngine baseXmlEngine = new BaseXmlEngine();
List<ScrutariSource> scrutariSourceList = scrutariSourceManager.getScrutariSourceList();
List<List<EngineStorage.Data>> list = new ArrayList<List<EngineStorage.Data>>();
List<List<EngineStorage.StoredData>> list = new ArrayList<List<EngineStorage.StoredData>>();
for (ScrutariSource scrutariSource : scrutariSourceManager.getScrutariSourceList()) {
String scrutariSourceName = scrutariSource.getScrutariSourceDef().getName();
BaseURI baseURI = aliasManager.getMainBaseURI(scrutariSourceName);
......@@ -322,7 +344,7 @@ public final class SctEngineImpl implements SctEngine {
list.add(null);
continue;
}
List<EngineStorage.Data> dataList = engineStorage.getDataList(scrutariSourceName);
List<EngineStorage.StoredData> dataList = engineStorage.getDataList(scrutariSourceName);
list.add(dataList);
updateLogHandler.setCurrentScrutariSource(scrutariSourceName);
timeLog.addSection(scrutariSourceName);
......@@ -334,7 +356,7 @@ public final class SctEngineImpl implements SctEngine {
}
Parameters parameters = new Parameters(dataCacheWriter, dataCoder, dataValidator, updateLogHandler, baseURI, codebase);
boolean first = true;
for (EngineStorage.Data data : dataList) {
for (EngineStorage.StoredData data : dataList) {
try (InputStream is = data.getInputStream()) {
baseXmlEngine.run(baseDataBuilder, parameters, is, first);
} catch (SAXException | IOException e) {
......@@ -357,7 +379,7 @@ public final class SctEngineImpl implements SctEngine {
IndexationXmlEngine indexationXmlEngine = new IndexationXmlEngine(dataCacheWriter, dataCoder, updateLogHandler);
int dataListSize = list.size();
for (int i = 0; i < dataListSize; i++) {
List<EngineStorage.Data> dataList = list.get(i);
List<EngineStorage.StoredData> dataList = list.get(i);
if (dataList == null) {
continue;
}
......@@ -366,7 +388,7 @@ public final class SctEngineImpl implements SctEngine {
BaseURI baseURI = aliasManager.getMainBaseURI(scrutariSourceName);
updateLogHandler.setCurrentScrutariSource(scrutariSourceName);
timeLog.addSection(scrutariSourceName);
for (EngineStorage.Data data : dataList) {
for (EngineStorage.StoredData data : dataList) {
try (InputStream is = data.getInputStream()) {
indexationXmlEngine.run(is, baseURI);
} catch (SAXException | IOException e) {
......@@ -376,6 +398,19 @@ public final class SctEngineImpl implements SctEngine {
}
timeLog.log("indexationdata_parsed");
}
for (int i = 0; i < dataListSize; i++) {
List<EngineStorage.StoredData> dataList = list.get(i);
if (dataList == null) {
continue;
}
for (EngineStorage.StoredData data : dataList) {
try {
data.backup();
} catch (IOException ioe) {
updateLogHandler.exception(ioe);
}
}
}
}
private void reloadConfDirectory() {
......@@ -415,12 +450,16 @@ public final class SctEngineImpl implements SctEngine {
errorType = 1;
}
}
if (errorType == 2) {
confState.setFileState(name, ConfConstants.FILE_XML_MALFORMED);
} else if (errorType == 1) {
confState.setFileState(name, ConfConstants.FILE_XML_ERROR);
} else {
confState.setFileState(name, ConfConstants.FILE_XML_WARNING);
switch (errorType) {
case 2:
confState.setFileState(name, ConfConstants.FILE_XML_MALFORMED);
break;
case 1:
confState.setFileState(name, ConfConstants.FILE_XML_ERROR);
break;
default:
confState.setFileState(name, ConfConstants.FILE_XML_WARNING);
break;
}
} else {
confState.setFileState(name, ConfConstants.FILE_OK);
......
......@@ -145,9 +145,9 @@ public final class CollectEngine {
private void fromInfoType(ScrutariSourceDef.UrlEntry urlEntry) throws StateErrorException {
URL infoUrl = urlEntry.getUrl();
ScrutariInfoContent scrutariInfoContent = getInfoContent(infoUrl, messageHandler);
FuzzyDate infoLastUpdate = scrutariInfoContent.getLastUpdate();
FuzzyDate infoDate = scrutariInfoContent.getDate();
if (!forceDataDownload) {
if (FuzzyDate.compare(infoLastUpdate, scrutariSource.getLastUpdate()) < 0) {
if (FuzzyDate.compare(infoDate, scrutariSource.getLastUpdate()) < 0) {
lastUpdateDate = FuzzyDate.max(lastUpdateDate, scrutariSource.getLastUpdate());
if (engineStorage.hasData(scrutariSourceName, infoUrl)) {
addNoChange(infoUrl);
......@@ -249,7 +249,7 @@ public final class CollectEngine {
}
private static ScrutariInfoContent readScrutariInfoDOM(Element element_xml, String urlRoot, MultiMessageHandler messageHandler) throws StateErrorException {
String lastDateString = null;
String dateString = null;
List<String> dataUrlStringList = new ArrayList<String>();
NodeList nodeList = element_xml.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++) {
......@@ -258,7 +258,7 @@ public final class CollectEngine {
Element el = (Element) node;
String tagname = el.getTagName();
if (tagname.equals("date")) {
lastDateString = XMLUtils.getData(el);
dateString = XMLUtils.getData(el);
} else if ((tagname.equals("path")) || (tagname.equals("scrutaridata-url"))) {
String pathString = XMLUtils.getData(el);
if (pathString.length() > 0) {
......@@ -268,7 +268,7 @@ public final class CollectEngine {
}
}
boolean wrongXml = false;
if (lastDateString == null) {
if (dateString == null) {
DomMessages.missingChildTag(messageHandler, element_xml.getTagName(), "date");
wrongXml = true;
}
......@@ -279,11 +279,11 @@ public final class CollectEngine {
if (wrongXml) {
throw new StateErrorException(CollectResult.INFO_WRONGXML_STATE);
}
FuzzyDate lastUpdate;
FuzzyDate date;
try {
lastUpdate = FuzzyDate.parse(lastDateString);
date = FuzzyDate.parse(dateString);
} catch (ParseException pe) {
DomMessages.wrongElementValue(messageHandler, "date", lastDateString);
DomMessages.wrongElementValue(messageHandler, "date", dateString);
throw new StateErrorException(CollectResult.INFO_WRONGXML_STATE);
}
List<URL> urlList = new ArrayList<URL>();
......@@ -296,7 +296,7 @@ public final class CollectEngine {
throw new StateErrorException(CollectResult.INFO_WRONGXML_STATE);
}
}
return new InternalScrutariInfoContent(lastUpdate, StringUtils.wrap(urlList.toArray(new URL[urlList.size()])));
return new InternalScrutariInfoContent(date, StringUtils.wrap(urlList.toArray(new URL[urlList.size()])));
}
private static URL buildDataUrl(String pathString, String urlRoot) throws MalformedURLException {
......@@ -316,17 +316,17 @@ public final class CollectEngine {
private static class InternalScrutariInfoContent implements ScrutariInfoContent {
private final FuzzyDate lastUpdate;
private final FuzzyDate date;
private final List<URL> urlList;
private InternalScrutariInfoContent(FuzzyDate lastUpdate, List<URL> urlList) {
this.lastUpdate = lastUpdate;
private InternalScrutariInfoContent(FuzzyDate date, List<URL> urlList) {
this.date = date;
this.urlList = urlList;
}
@Override
public FuzzyDate getLastUpdate() {
return lastUpdate;
public FuzzyDate getDate() {
return date;
}
@Override
......@@ -348,15 +348,11 @@ public final class CollectEngine {
}
@Override
public InputStream getInputStream() {
public InputStream getInputStream() throws IOException {
if (tempFile == null) {
return null;
}
try {
return new FileInputStream(tempFile);
} catch (IOException ioe) {
throw new SctStorageException(ioe);
}
return new FileInputStream(tempFile);
}
@Override
......
......@@ -9,6 +9,8 @@
package fr.exemole.sctserver.tools.log;
import java.util.HashSet;
import java.util.Set;
import net.mapeadores.util.localisation.LocalisationUtils;
import net.mapeadores.util.localisation.Message;
import net.mapeadores.util.logging.MessageLog;
......@@ -25,19 +27,23 @@ import org.xml.sax.SAXException;
public class UpdateLogHandler implements MultiMessageHandler {
private final MessageLogBuilder builder = new MessageLogBuilder();
private final Set<String> exceptionOnlySet = new HashSet<String>();
private String currentURI;
private String currentKey;
private int keyOccurrence = 0;
private boolean logExceptionOnly = false;
public UpdateLogHandler() {
}
public void setCurrentScrutariSource(String name) {
flush();
builder.setCurrentURI(name);
setCurrentURI(name);
}
public void exception(Exception e) {
flush();
this.logExceptionOnly = true;
exceptionOnlySet.add(currentURI);
if (e instanceof SAXException) {
DomMessages.saxException(builder, (SAXException) e);
} else {
......@@ -47,21 +53,29 @@ public class UpdateLogHandler implements MultiMessageHandler {
@Override
public void addMessage(String category, Message message) {
if (currentKey != null) {
if (message.getMessageKey().equals(currentKey)) {
keyOccurrence++;
} else {
flush();
if (!logExceptionOnly) {
if (currentKey != null) {
if (message.getMessageKey().equals(currentKey)) {
keyOccurrence++;
} else {
flush();
}
}
if (keyOccurrence < 11) {
builder.addMessage(category, message);
}
}
if (keyOccurrence < 11) {
builder.addMessage(category, message);
}
}
@Override
public void setCurrentURI(String uri) {
flush();
this.currentURI = uri;
if (exceptionOnlySet.contains(uri)) {
this.logExceptionOnly = true;
} else {
this.logExceptionOnly = false;
}
builder.setCurrentURI(uri);
}
......@@ -78,5 +92,4 @@ public class UpdateLogHandler implements MultiMessageHandler {
this.keyOccurrence = 0;
}
}
/* SctServer_Directory - Copyright (c) 2008-2017 Vincent Calame - Exemole
/* SctServer_Directory - Copyright (c) 2008-2018 Vincent Calame - Exemole
* Logiciel libre donné sous triple licence :
* 1) selon les termes de la CeCILL V2
* 2) selon les termes de l’EUPL V.1.1
......@@ -43,6 +43,7 @@ import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.RandomAccess;
......@@ -75,7 +76,7 @@ import net.scrutari.db.tools.AliasManager;
*/
public class DirectoryEngineStorage implements EngineStorage {
private final static List<Data> EMPTY_DATALIST = Collections.emptyList();
private final static List<StoredData> EMPTY_DATALIST = Collections.emptyList();
private final static String URLMAP_FILENAME = "urlmap.txt";
private final File confDir;
private final File varDir;
......@@ -260,14 +261,14 @@ public class DirectoryEngineStorage implements EngineStorage {
}
@Override
public List<Data> getDataList(String scrutariSourceName) {
public List<StoredData> getDataList(String scrutariSourceName) {
File dir = new File(dataDirectory, scrutariSourceName);
if (!dir.exists()) {
return EMPTY_DATALIST;
}
SortedMap<Integer, FileData> map = getDataFileMap(dir);
EngineStorage.Data[] array = map.values().toArray(new EngineStorage.Data[map.size()]);
return new EngineStorageDataList(array);
EngineStorage.StoredData[] array = map.values().toArray(new EngineStorage.StoredData[map.size()]);
return new StoredDataList(array);
}
@Override
......@@ -303,7 +304,7 @@ public class DirectoryEngineStorage implements EngineStorage {
if (currentNumber != null) {
FileData currentFileData = currentDataFileMap.get(currentNumber);
if (currentFileData != null) {
currentFileData.file.renameTo(tempDataFile);
currentFileData.update(tempDataFile);
tempDataFileList.add(tempDataFile);
}
}
......@@ -317,15 +318,28 @@ public class DirectoryEngineStorage implements EngineStorage {
}
buf.append(newNumber).append("=").append(url.toString()).append("\n");
}
for (File f : dir.listFiles()) {
if (f.getName().startsWith("data-")) {
f.delete();
}
}
Set<String> prefixSet = new HashSet<String>();
for (File f : tempDataFileList) {
String name = f.getName();
String newName = "data-" + name.substring(9) + ".xml";
f.renameTo(new File(dir, newName));
String prefix = "data-" + name.substring(9);
prefixSet.add(prefix);
f.renameTo(new File(dir, prefix + ".xml"));
}
for (File f : dir.listFiles()) {
String name = f.getName();
if (name.startsWith("data-")) {
int idx = name.indexOf(".");
boolean delete = true;
if (idx != -1) {
String prefix = name.substring(0, idx);
if (prefixSet.contains(prefix)) {
delete = false;
}
}
if (delete) {
f.delete();
}
}
}
try {
FileUtils.writeStringToFile(new File(dir, URLMAP_FILENAME), buf.toString(), "UTF-8");
......@@ -522,7 +536,9 @@ public class DirectoryEngineStorage implements EngineStorage {
String name = f.getName();
Matcher matcher = pattern.matcher(name);
if (matcher.matches()) {
map.put(Integer.parseInt(matcher.group(1)), new FileData(f));
File backupFile = new File(directory, name + ".bak");
File errorFile = new File(directory, name + ".err");
map.put(Integer.parseInt(matcher.group(1)), new FileData(f, backupFile, errorFile));
}
}
return map;
......@@ -589,11 +605,11 @@ public class DirectoryEngineStorage implements EngineStorage {
}
private static class EngineStorageDataList extends AbstractList<EngineStorage.Data> implements RandomAccess {
private static class StoredDataList extends AbstractList<EngineStorage.StoredData> implements RandomAccess {
private final EngineStorage.Data[] array;
private final EngineStorage.StoredData[] array;
private EngineStorageDataList(EngineStorage.Data[] array) {
private StoredDataList(EngineStorage.StoredData[] array) {
this.array = array;
}
......@@ -603,28 +619,58 @@ public class DirectoryEngineStorage implements EngineStorage {
}
@Override
public EngineStorage.Data get(int index) {
public EngineStorage.StoredData get(int index) {
return array[index];
}
}
private static class FileData implements EngineStorage.Data {
private static class FileData implements EngineStorage.StoredData {
private final File file;
private final File backupFile;
private final File errorFile;
private FileData(File file) {
private FileData(File file, File backupFile, File errorFile) {
this.file = file;
this.backupFile = backupFile;
this.errorFile = errorFile;
}
@Override
public InputStream getInputStream() {
try {
return new FileInputStream(file);
} catch (IOException ioe) {
throw new SctStorageException(ioe);
public InputStream getInputStream() throws IOException {
return new FileInputStream(file);
}
@Override
public boolean hasBackup() {
return ((backupFile.exists()) && (!backupFile.isDirectory()));
}
@Override
public boolean restoreBackup() throws IOException {
if (!hasBackup()) {
return false;
}
FileUtils.copyFile(file, errorFile);
FileUtils.copyFile(backupFile, file);
return true;
}
@Override
public void backup() throws IOException {
if (backupFile.exists()) {
FileUtils.deleteQuietly(backupFile);
}
FileUtils.copyFile(file, backupFile);
}
public void update(File tmpFile) {
if (errorFile.exists()) {
FileUtils.deleteQuietly(errorFile);
}
file.renameTo(tmpFile);
}
}
......
/* BdfServer - Copyright (c) 2014 -2018 Vincent Calame - Exemole
/* UtilLib - Copyright (c) 2014 -2018 Vincent Calame - Exemole
* Logiciel libre donné sous triple licence :
* 1) selon les termes de la CeCILL V2
* 2) selon les termes de l’EUPL V.1.1
......
/* UtilLib - Copyright (c) 2018 Vincent Calame - Exemole
* Logiciel libre donné sous triple licence :
* 1) selon les termes de la CeCILL V2
* 2) selon les termes de l’EUPL V.1.1
* 3) selon les termes de la GNU GPLv3
* Voir le fichier licences.txt
*/
package net.mapeadores.util.json;
/**
*
* @author Vincent Calame
*/
public interface PropertyEligibility {
public boolean includeProperty(String propertyName);
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment