Commit ba96d6b2 authored by Denis Salem's avatar Denis Salem
Browse files

still replacing PreProcessor

parent 15397344
......@@ -145,7 +145,7 @@
DONE | Update command help with.
DONE | Handle ftp encoding error.
DONE | Sort by metadata.
DONE | Speed up blog exportation (almost five times faster!).
WIP | Speed up blog exportation (almost five times faster!).
WIP | Chapters engine.
TODO | Open with navigator manual.
TODO | Client-side search engine.
......
......@@ -38,7 +38,7 @@ from venc2.patterns.latex2mathml import Latex2MathML
from venc2.patterns.contextual import extra_contextual_pattern_names
from venc2.patterns.non_contextual import non_contextual_pattern_names
from venc2.patterns.processor import Processor
from venc2.patterns.processor import PreProcessor
from venc2.patterns.processor import ProcessedString
from venc2.threads.categories import CategoriesThread
from venc2.threads.dates import DatesThread
from venc2.threads.entries import EntriesThread
......@@ -172,11 +172,19 @@ def export_blog(argv=list()):
except AttributeError:
markup_language = datastore.blog_configuration["markup_language"]
entry.preview = PreProcessor(processor.batch_process(entry.preview, entry.filename).process_markup_language(markup_language, entry.filename))
entry.content = PreProcessor(processor.batch_process(entry.content, entry.filename).process_markup_language(markup_language, entry.filename))
processor.process(entry.preview)
entry.preview.process_markup_language(markup_language, entry.filename)
processor.process(entry.content, entry.filename)
entry.content.process_markup_language(markup_language, entry.filename)
print(entry.content.string, entry.preview.string)
die("end of test")
entry.html_wrapper = deepcopy(theme.entry)
entry.html_wrapper.above = PreProcessor(''.join(processor.batch_process(entry.html_wrapper.above, "entry.html", False).sub_strings))
entry.html_wrapper.below = PreProcessor(''.join(processor.batch_process(entry.html_wrapper.below, "entry.html", False).sub_strings))
entry.html_wrapper.above = ProcessedString(''.join(processor.batch_process(entry.html_wrapper.above, "entry.html", False).sub_strings))
entry.html_wrapper.below = ProcessedString(''.join(processor.batch_process(entry.html_wrapper.below, "entry.html", False).sub_strings))
entry.rss_wrapper = deepcopy(theme.rss_entry)
entry.rss_wrapper.above = PreProcessor(''.join(processor.batch_process(entry.rss_wrapper.above, "rssEntry.xml", False).sub_strings))
......
......@@ -30,6 +30,7 @@ from venc2.datastore.metadata import build_categories_tree
from venc2.datastore.metadata import MetadataNode
from venc2.datastore.metadata import Chapter
from venc2.helpers import notify
from venc2.l10n import messages
from venc2.patterns.non_contextual import get_embed_content
def merge(iterable, argv):
......@@ -37,6 +38,7 @@ def merge(iterable, argv):
class DataStore:
def __init__(self):
notify(messages.loading_data)
self.blog_configuration = get_blog_configuration()
self.disable_threads = [thread_name.strip() for thread_name in self.blog_configuration["disable_threads"].split(',')]
self.entries = list()
......@@ -116,7 +118,6 @@ class DataStore:
return getattr(entry, self.blog_configuration["sort_by"])
except AttributeError:
print("FAILED")
return ''
def set_max_category_weight(self, value):
......
......@@ -30,7 +30,7 @@ from venc2.helpers import notify
from venc2.l10n import messages
from venc2.datastore.metadata import build_categories_tree
from venc2.datastore.metadata import MetadataNode
from venc2.patterns.processor import PreProcessor
from venc2.patterns.processor import ProcessedString
class EntryWrapper:
def __init__(self, wrapper, filename):
......@@ -45,8 +45,8 @@ class EntryWrapper:
if p in w[0] or p in w[1]:
die(messages.too_much_call_of_content.format(filename))
self.above = PreProcessor(w[0])
self.below = PreProcessor(w[1])
self.above = ProcessedString(w[0], filename)
self.below = ProcessedString(w[1], filename)
self.required_content_pattern = pattern
return
......@@ -67,8 +67,8 @@ class Entry:
if len(entry_parted) == 2:
entry_parted = [entry_parted[0]] + entry_parted[1].split("---VENC-END-PREVIEW---\n")
if len(entry_parted) == 3:
self.preview = PreProcessor(entry_parted[1])
self.content = PreProcessor(entry_parted[2])
self.preview = ProcessedString(entry_parted[1], filename)
self.content = ProcessedString(entry_parted[2], filename)
try:
metadata = yaml.load(entry_parted[0])
......
......@@ -20,7 +20,7 @@
from venc2.datastore.entry import EntryWrapper
from venc2.helpers import die
from venc2.l10n import messages
from venc2.patterns.processor import PreProcessor
from venc2.patterns.processor import ProcessedString
themes_descriptor = {
"dummy": {"columns":1,"_themeDescription_": messages.theme_description_dummy},
......@@ -34,12 +34,12 @@ class Theme:
non_contextual_pattern_names["audio"] = self.get_audio
try:
self.header = PreProcessor(open(theme_folder+"chunks/header.html",'r').read())
self.footer = PreProcessor(open(theme_folder+"chunks/footer.html",'r').read())
self.rss_header = PreProcessor(open(theme_folder+"chunks/rssHeader.xml",'r').read())
self.rss_footer = PreProcessor(open(theme_folder+"chunks/rssFooter.xml",'r').read())
self.atom_header = PreProcessor(open(theme_folder+"chunks/atomHeader.xml",'r').read())
self.atom_footer = PreProcessor(open(theme_folder+"chunks/atomFooter.xml",'r').read())
self.header = ProcessedString(open(theme_folder+"chunks/header.html",'r').read(), "header.html")
self.footer = ProcessedString(open(theme_folder+"chunks/footer.html",'r').read(), "footer.html")
self.rss_header = ProcessedString(open(theme_folder+"chunks/rssHeader.xml",'r').read(), "rssHeader.html")
self.rss_footer = ProcessedString(open(theme_folder+"chunks/rssFooter.xml",'r').read(), "rssFooter.html")
self.atom_header = ProcessedString(open(theme_folder+"chunks/atomHeader.xml",'r').read(), "atomHeader.html")
self.atom_footer = ProcessedString(open(theme_folder+"chunks/atomFooter.xml",'r').read(), "atomFooter.html")
self.entry = EntryWrapper(open(theme_folder+"chunks/entry.html",'r').read(), "entry.html")
self.rss_entry = EntryWrapper(open(theme_folder+"chunks/rssEntry.xml",'r').read(),"rssEntry.xxml")
......
......@@ -96,3 +96,6 @@ class Messages:
invalid_or_missing_metadata = "\"{0}\": Invalid value or missing metadata in entry \"{1}\"."
generating_rss = "Generating RSS feed..."
generating_atom = "Generating Atom feed..."
malformed_patterns_missing_closing_symbols = "Malformed patterns in \"{0}\": One or more closing symbols are missing."
malformed_patterns_missing_opening_symbols = "Malformed patterns in \"{0}\": One or more closing symbols are missing."
loading_data = "Loading data..."
......@@ -94,5 +94,8 @@ class Messages:
invalid_or_missing_metadata = "\"{0}\": Valeur de meta-donnée invalide ou abstente dans la publication \"{1}\"."
generating_rss = "Génération du flux RSS..."
generating_atom = "Génération du flux Atom..."
malformed_patterns_missing_closing_symbols = "Motifs mal formés dans \"{0}\": Il manque un ou plusieurs symboles de fermeture."
malformed_patterns_missing_opening_symbols = "Motifs mal formés dans \"{0}\": Il manque un ou plusieurs symboles d'ouverture."
loading_data = "Chargement des données..."
......@@ -29,16 +29,22 @@ from docutils.utils import SystemMessage
from venc2.helpers import get_formatted_message
from venc2.helpers import highlight_value
from venc2.helpers import notify
from venc2.helpers import die
from venc2.helpers import remove_by_value
from venc2.helpers import PatternInvalidArgument
from venc2.helpers import GenericMessage
from venc2.l10n import messages
# Special case of KeyError
class UnknownContextual(KeyError):
pass
markup_language_errors = []
def cgi_escape(string):
return cgi.escape(string).encode(
'ascii',
'xmlcharrefreplace'
).decode(
encoding='ascii'
)
def get_markers_indexes(string):
op = []
cp = []
......@@ -75,21 +81,80 @@ def handle_markup_language_error(message, line=None, string=None):
else:
print(lines[lineno])
class PreProcessor():
def __init__(self, string):
def parse_markup_language(string, markup_language, ressource):
if markup_language == "Markdown":
string = markdown.markdown(string)
elif markup_language == "reStructuredText":
string = publish_parts(string, writer_name='html', settings_overrides={'doctitle_xform':False, 'halt_level': 2, 'traceback': True, "warning_stream":"/dev/null"})['html_body']
elif markup_language != "none":
err = messages.unknown_markup_language.format(markup_language, ressource)
handle_markup_language_error(err)
return string
# Special case of KeyError
class UnknownContextual(KeyError):
pass
class ProcessedString():
def __init__(self, string, ressource):
self.open_pattern_pos, self.close_pattern_pos = get_markers_indexes(string)
self.len_open_pattern_pos = len(self.open_pattern_pos)
self.len_close_pattern_pos = len(self.close_pattern_pos)
if self.len_open_pattern_pos != self.len_close_pattern_pos:
raise Exception
if self.len_open_pattern_pos > self.len_close_pattern_pos:
l = self.open_pattern_pos + self.close_pattern_pos
mn, mx = min(l), max(l)
die(messages.malformed_patterns_missing_closing_symbols.format(source))
elif self.len_open_pattern_pos > self.len_close_pattern_pos:
l = self.open_pattern_pos + self.close_pattern_pos
mn, mx = min(l), max(l)
die(messages.malformed_patterns_missing_closing_symbols.format(source))
self.string = string
self.ressource = ressource
self.keep_appart_from_markup_indexes = list()
self.keep_appart_from_markup_inc = 0
def keep_appart_from_markup_indexes_append(paragraphe):
self.keep_appart_from_markup_indexes.append((self.keep_appart_from_markup_inc, paragraphe))
def keep_appart_from_markup_indexes_append(paragraphe, new_chunk, escape):
if escape:
new_chunk = cgi_escape(new_chunk)
self.keep_appart_from_markup_indexes.append((self.keep_appart_from_markup_inc, paragraphe, new_chunk))
output = "VENC_TEMPORARY_REPLACEMENT_"+str(self.keep_appart_from_markup_inc)
self.keep_appart_from_markup_inc +=1
return output
def process_markup_language(self, markup_language, source):
try:
self.string = parse_markup_language(self.string, markup_language, source)
# catch error from reStructuredText
except SystemMessage as e:
try:
line = int(str(e).split(':')[1])
msg = str(e).split(':')[2].strip()
handle_markup_language_error(source+": "+msg, line=line, string=string)
except Exception as ee:
print(ee, len(string.split('\n')))
handle_markup_language_error(source+", "+str(e))
for triplet in self.keep_appart_from_markup_indexes:
index, paragraphe, new_chunk = triplet
if paragraphe:
self.string = self.string.replace(
"<p>VENC_TEMPORARY_REPLACEMENT_"+str(index)+"</p>",
new_chunk
)
else:
self.string = self.string.replace(
"VENC_TEMPORARY_REPLACEMENT_"+str(index),
new_chunk
)
"""
class PreProcessor0ld():
def __init__(self, string):
self.blacklist = list()
......@@ -167,19 +232,7 @@ class PreProcessor0ld():
)
return string
def parse_markup_language(string, markup_language, source):
if markup_language == "Markdown":
string = markdown.markdown(string)
elif markup_language == "reStructuredText":
string = publish_parts(string, writer_name='html', settings_overrides={'doctitle_xform':False, 'halt_level': 2, 'traceback': True, "warning_stream":"/dev/null"})['html_body']
elif markup_language != "none":
err = messages.unknown_markup_language.format(markup_language, source)
handle_markup_language_error(err)
return string
"""
class Processor():
def __init__(self):
......@@ -188,7 +241,6 @@ class Processor():
self.current_input_string = str()
self.ressource = str()
self.errors = list()
self.clean_after_and_before = list()
# Some patterns are contextual while others are constant in time.
# To save time we wan't to perform two-pass analysis. In the first one
......@@ -354,13 +406,13 @@ class Processor():
return pre_processed
"""
def process(self, pre_processed, ressource, escape=False):
def process(self, pre_processed, escape=False):
op, cp, string, lo, lc = pre_processed.open_pattern_pos, pre_processed.close_pattern_pos, pre_processed.string, pre_processed.len_open_pattern_pos, pre_processed.len_close_pattern_pos
if lo == 0 and lc == 0:
return pre_processed
self.ressource = ressource
self.ressource = pre_processed.ressource
while lo:
diff = 18446744073709551616
i = 0
......@@ -388,20 +440,23 @@ class Processor():
""" réintégrer l'exclusi0n de certain pattern, réintégrer Escape N0Escape """
if (not current_pattern in self.blacklist) and (not current_pattern in self.forbidden) :
if current_pattern in ["CodeHighlight", "Latex2MathML", "IncludeFile", "audio", "video","EmbedContent"]:
pre_processed.keep_appart_from_markup_index_append(True)
new_chunk = pre_processed.keep_appart_from_markup_index_append(
True,
self.run_pattern(current_pattern, fields[1:]),
escape
)
elif current_pattern in ["SetColor"]:
pre_processed.keep_appart_from_markup_index_append(False))
new_chunk = pre_processed.keep_appart_from_markup_index_append(
False,
self.run_pattern(current_pattern, fields[1:]),
escape,
)
else:
new_chunk = self.run_pattern(current_pattern, fields[1:])
if escape:
new_chunk = cgi.escape(new_string).encode(
'ascii',
'xmlcharrefreplace'
).decode(
encoding='ascii'
)
new_chunk = self.run_pattern(current_pattern, fields[1:])
if escape:
new_chunk = cgi_escape(new_string)
string = string[0:vop] + new_chunk + string[vcp+2:]
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment