Commit 53ec5129 authored by Denis Salem's avatar Denis Salem
Browse files

fix CodeHighlight

parent 8b45d3e8
......@@ -11,6 +11,8 @@ dateFormat: '%A %d. %B %Y'
entriesPerPages: 5
ftpHost: Nom d'hôte FTP
license: CC By-SA
codeHighlightCssOverride: true
markup_language: 'Markdown'
path:
categoryDirectoryName: '{category}'
datesDirectoryName: ''
......
......@@ -5,3 +5,73 @@ title:
style: features
---
## V __en__ C is super easy to use and learn!
.:GetRandomNumber::0::10::3:.
.:CodeHighlight::python::False::# Copyright 2016, 2018 Denis Salem
#
# This file is part of VenC.
#
# VenC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# VenC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
import os
import pygments.lexers
import pygments.formatters
from venc2.helpers import die
from venc2.helpers import notify
from venc2.l10n import messages
class CodeHighlight:
def __init__(self, override):
self._override = override
self._includes = dict()
def get_style_sheets(self, argv=list()):
output = str()
for filename in self._includes.keys():
output += "<link rel=\"stylesheet\" href=\".:GetRelativeOrigin:."+filename+"\" type=\"text/css\" />\n"
return output
def export_style_sheets(self):
extra = os.listdir(os.getcwd()+"/extra/")
for key in self._includes:
if key not in extra or self._override:
stream = open(os.getcwd()+"/extra/"+key,'w')
stream.write(self._includes[key])
def highlight(self, argv):
try:
name = "venc_source_"+argv[0].replace('+','Plus')
lexer = pygments.lexers.get_lexer_by_name(argv[0], stripall=True)
formatter = pygments.formatters.HtmlFormatter(linenos=("table" if argv[1]=="True" else False), cssclass=name)
code = argv[2]
result = pygments.highlight(code.replace("\:",":"), lexer, formatter)
css = formatter.get_style_defs()
if not name+".css" in self._includes.keys():
self._includes[name+".css"] = css
return result
except pygments.util.ClassNotFound:
die(messages.unknown_language.format(argv[0]))
:.
some paragraphe
### sub title
</div>
<footer id="__VENC_NAVIGATION__">
.:GetPreviousPage:: <a id="previous" href="{0[destinationPageUrl]} "></a>:.
<ul id="pagesList">.:PagesList::5::<li><a href="{0[pageUrl]}">{0[pageNumber]}</a></li>:: . :.</ul>
<ul id="pagesList">.:ForPages::5::<li><a href="{0[pageUrl]}">{0[pageNumber]}</a></li>:: . :.</ul>
.:GetNextPage:: <a id="next" href="{0[destinationPageUrl]} "></a>:.
</footer>
</body>
......
......@@ -12,6 +12,7 @@
</script>
<link rel="alternate" type="application/rss+xml" title=".:GetBlogName:." href="feed.xml" />
<link rel="stylesheet" href=".:GetRelativeOrigin:.style.css" type="text/css" />
.:GetStyleSheets:.
.:IfInThread::<link rel="stylesheet" href=".:GetRelativeOrigin:.styleThread.css" type="text/css" />::<link rel="stylesheet" href=".:GetRelativeOrigin:.styleEntry.css" type="text/css" />:.
</head>
<body>
......
......@@ -105,10 +105,15 @@
DONE | Add random number generator.
DONE | Add function Include. (Add a directory force included ressources)
DONE | Create theme folder if missing while setup new theme.
WIP | Massive refactorisation, code cleaning and optimisation.
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
WIP | Auto include CodeHighlight CSS.
TODO | Fix vertical scroll bar in code snippet.
DONE | Auto include CodeHighlight CSS.
DONE | Add CodeHighlight css override option in blog configuration.
DONE | Raise error if unknown markup language required.
DONE | Fix markup language collision AGAIN...
DONE | Massive refactorisation, code cleaning and optimisation.
DONE | Fix pattern recursion issue (CodeHighlight).
DONE | Fix vertical scroll bar in code snippet.
WIP | Add reStructuredText support, remove do_not_use_markdown option.
TODO | Change nomenclature: Most of the usual patterns are changed. Fix doc.
TODO | Add relevant templates for themes.
TODO | Add categories tree in entry.
TODO | Reorganize themes dependencies in setup (adding default templates, scripts, and pages to include).
......
......@@ -12,6 +12,7 @@
</script>
<link rel="alternate" type="application/rss+xml" title=".:GetBlogName:." href="feed.xml" />
<link rel="stylesheet" href=".:GetRelativeOrigin:.style.css" type="text/css" />
.:GetStyleSheets:.
.:IfInThread::<link rel="stylesheet" href=".:GetRelativeOrigin:.styleThread.css" type="text/css" />::<link rel="stylesheet" href=".:GetRelativeOrigin:.styleEntry.css" type="text/css" />:.
</head>
<body>
......
......@@ -26,7 +26,6 @@ import base64
import shutil
import subprocess
from venc2.datastore.configuration import get_blog_configuration
from venc2.datastore.datastore import DataStore
from venc2.datastore.theme import themes_descriptor
from venc2.datastore.theme import Theme
......@@ -37,13 +36,16 @@ from venc2.l10n import messages
from venc2.patterns.code_highlight import CodeHighlight
from venc2.patterns.contextual import extra_contextual_pattern_names
from venc2.patterns.non_contextual import non_contextual_pattern_names
from venc2.patterns.processor import merge_batches
from venc2.patterns.processor import Processor
from venc2.patterns.processor import PreProcessor
from venc2.threads.categories_thread import CategoriesThread
from venc2.threads.dates_thread import DatesThread
from venc2.threads.main_thread import MainThread
# Initialisation of environment
datastore = DataStore()
code_highlight = CodeHighlight(datastore.blog_configuration["codeHighlightCssOverride"])
non_contextual_pattern_names_datastore = {
# General entry data
"GetEntryTitle" : "get_entry_title",
......@@ -79,8 +81,7 @@ non_contextual_pattern_names_datastore = {
}
non_contextual_pattern_names_code_highlight = {
"CodeHighlight" : "highlight",
"GetStyleSheets" : "get_style_sheets"
"CodeHighlight" : code_highlight.highlight
}
contextual_pattern_names = {
......@@ -90,6 +91,7 @@ contextual_pattern_names = {
"GetNextPage" : "get_next_page",
"GetPreviousPage" : "get_previous_page",
"ForPages" : "for_pages",
"GetStyleSheets" : code_highlight.get_style_sheets,
**extra_contextual_pattern_names
}
......@@ -99,9 +101,6 @@ def export_and_remote_copy(argv=list()):
remote_copy()
def export_blog(argv=list()):
# Initialisation of environment
datastore = DataStore()
# Initialisation of theme
theme_folder = "theme/"
......@@ -118,7 +117,6 @@ def export_blog(argv=list()):
theme = Theme(theme_folder)
code_highlight = CodeHighlight()
# Set up of non-contextual patterns
......@@ -128,7 +126,7 @@ def export_blog(argv=list()):
processor.set_function(pattern_name, getattr(datastore, non_contextual_pattern_names_datastore[pattern_name]))
for pattern_name in non_contextual_pattern_names_code_highlight.keys():
processor.set_function(pattern_name, getattr(code_highlight, non_contextual_pattern_names_code_highlight[pattern_name]))
processor.set_function(pattern_name, non_contextual_pattern_names_code_highlight[pattern_name])
for pattern_name in non_contextual_pattern_names.keys():
processor.set_function(pattern_name, non_contextual_pattern_names[pattern_name])
......@@ -137,32 +135,29 @@ def export_blog(argv=list()):
for pattern_name in contextual_pattern_names.keys():
processor.blacklist.append(pattern_name)
for pattern_name in contextual_pattern_names.keys():
processor.blacklist.append(pattern_name)
""" Ugly piece of code """
# List of patterns where we want to remove <p></p> tag
processor.clean_after_and_before.append("CodeHighlight")
notify(messages.pre_process)
# Now we want to perform first parsing pass on entries and chunk
for entry in datastore.get_entries():
# Every chunks are preprocessed again because of contextual patterns
entry.content = PreProcessor(merge_batches(processor.batch_process(entry.content, entry.filename, not entry.do_not_use_markdown)))
try:
markup_language = getattr(entry, "markup_language")
except AttributeError:
markup_language = datastore.blog_configuration["markup_language"]
entry.content = PreProcessor(processor.batch_process(entry.content, entry.filename).process_markup_language(markup_language, entry.filename))
entry.html_wrapper = deepcopy(theme.entry)
entry.html_wrapper.above = PreProcessor(merge_batches(processor.batch_process(entry.html_wrapper.above, "entry.html", not entry.do_not_use_markdown)))
entry.html_wrapper.below = PreProcessor(merge_batches(processor.batch_process(entry.html_wrapper.below, "entry.html",not entry.do_not_use_markdown)))
entry.html_wrapper.above = PreProcessor(''.join(processor.batch_process(entry.html_wrapper.above, "entry.html", False).sub_strings))
entry.html_wrapper.below = PreProcessor(''.join(processor.batch_process(entry.html_wrapper.below, "entry.html", False).sub_strings))
entry.rss_wrapper = deepcopy(theme.rss_entry)
entry.rss_wrapper.above = PreProcessor(merge_batches(processor.batch_process(entry.rss_wrapper.above, "rssEntry.html", not entry.do_not_use_markdown)))
entry.rss_wrapper.below = PreProcessor(merge_batches(processor.batch_process(entry.rss_wrapper.below, "rssEntry.html", not entry.do_not_use_markdown)))
entry.rss_wrapper.above = PreProcessor(''.join(processor.batch_process(entry.rss_wrapper.above, "rssEntry.html", False).sub_strings))
entry.rss_wrapper.below = PreProcessor(''.join(processor.batch_process(entry.rss_wrapper.below, "rssEntry.html", False).sub_strings))
theme.header = PreProcessor(merge_batches(processor.batch_process(theme.header, "header.html")))
theme.footer = PreProcessor(merge_batches(processor.batch_process(theme.footer, "footer.html")))
theme.rssHeader = PreProcessor(merge_batches(processor.batch_process(theme.rss_header, "rssHeader.html")))
theme.rssFooter = PreProcessor(merge_batches(processor.batch_process(theme.rss_footer, "rssFooter.html")))
theme.header = PreProcessor(''.join(processor.batch_process(theme.header, "header.html").sub_strings))
theme.footer = PreProcessor(''.join(processor.batch_process(theme.footer, "footer.html").sub_strings))
theme.rssHeader = PreProcessor(''.join(processor.batch_process(theme.rss_header, "rssHeader.html").sub_strings))
theme.rssFooter = PreProcessor(''.join(processor.batch_process(theme.rss_footer, "rssFooter.html").sub_strings))
# cleaning directory
shutil.rmtree("blog", ignore_errors=False, onerror=rm_tree_error_handler)
......@@ -201,18 +196,16 @@ def copy_recursively(src, dest):
def edit_and_export(argv):
blog_configuration = get_blog_configuration()
if len(argv) != 1:
die(Messages.missing_params.format("--edit-and-export"))
try:
proc = subprocess.Popen([blog_configuration["textEditor"], argv[0]])
proc = subprocess.Popen([datastore.blog_configuration["textEditor"], argv[0]])
while proc.poll() == None:
pass
except TypeError:
die(Messages.unknown_text_editor.format(blog_configuration["textEditor"]))
die(Messages.unknown_text_editor.format(datastore.blog_configuration["textEditor"]))
except:
raise
......
......@@ -118,6 +118,7 @@ def new_blog(argv):
"ftpHost": messages.ftp_host,
"blogLanguage": messages.blog_language,
"authorEmail": messages.your_email,
"codeHighlightCssOverride": False,
"path": {"ftp": messages.ftp_path,
"indexFileName": "index{0[pageNumber]}.html",
"categoryDirectoryName": "{category}",
......@@ -127,7 +128,8 @@ def new_blog(argv):
"entriesPerPages": 10,
"columns": 1,
"rssThreadLenght": 5,
"reverseThreadOrder": True}
"reverseThreadOrder": True,
"markup_language": "markdown"}
for folder_name in argv:
try:
os.mkdir(folder_name)
......
......@@ -44,7 +44,8 @@ def get_blog_configuration():
"entriesPerPages",
"columns",
"rssThreadLenght",
"reverseThreadOrder"
"reverseThreadOrder",
"markup_language"
]
everything_is_okay = True
......@@ -67,6 +68,10 @@ def get_blog_configuration():
everything_is_okay = False
notify(messages.missing_mandatory_field_in_blog_conf.format(field),"RED")
if not blog_configuration["markup_language"] in ["none", "Markdown"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blogConfiguration.yaml"),"RED")
if not everything_is_okay:
exit()
......
......@@ -18,7 +18,6 @@
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
import datetime
import markdown
import os
import time
import yaml
......@@ -61,16 +60,9 @@ class Entry:
# Setting up optional metadata
for key in metadata.keys():
if not key in ["authors","tags","categories","entry_name","doNotUseMarkdown"]:
if not key in ["authors","tags","categories","entry_name"]:
setattr(self, key, metadata[key])
# Are we using markdown?
if "doNotUseMarkdown" in metadata.keys():
self.do_not_use_markdown = True
else:
self.do_not_use_markdown = False
raw_content = markdown.markdown(raw_content)
self.filename = filename
self.id = filename.split('__')[0]
......
......@@ -81,4 +81,5 @@ class Messages:
variable_error_in_filename="Variable error in main configuration file where paths and filenames are defined: {0} doesn't exists."
wrong_pattern_argument="Argument '{0}' = '{1}' from pattern '{2}' is wrong."
pattern_argument_must_be_integer="It must be a positive integer."
wrong_permissions = "{0} : You don't have the right permissions on this file."
wrong_permissions = "{0} : You don't have the right permissions on this file in {1}."
unknown_markup_language = "{0}: Unsupported markup language."
......@@ -80,4 +80,5 @@ class Messages:
wrong_pattern_argument="L'argument '{0}' = '{1}' du motif '{2}' n'est pas correct."
pattern_argument_must_be_integer="L'argument doit être un entier positif."
wrong_permissions = "{0} : Vous n'avez pas les bonnes permissions d'accès sur ce fichier."
unknown_markup_language = "{0}: Langage de balisage non supporté dans {1}."
......@@ -26,20 +26,21 @@ from venc2.helpers import notify
from venc2.l10n import messages
class CodeHighlight:
def __init__(self):
def __init__(self, override):
self._override = override
self._includes = dict()
def get_style_sheets(self, argv=list()):
output = str()
for filename in self._includes.keys():
output += "<link rel=\"stylesheet\" href=\".:GetRelativeOrigin:."+filename+" type=\"text/css\" />\n"
output += "<link rel=\"stylesheet\" href=\".:GetRelativeOrigin:."+filename+"\" type=\"text/css\" />\n"
return output
def export_style_sheets(self):
extra = os.listdir(os.getcwd()+"/extra/")
for key in self._includes:
if key not in extra:
if key not in extra or self._override:
stream = open(os.getcwd()+"/extra/"+key,'w')
stream.write(self._includes[key])
......@@ -49,10 +50,19 @@ class CodeHighlight:
lexer = pygments.lexers.get_lexer_by_name(argv[0], stripall=True)
formatter = pygments.formatters.HtmlFormatter(linenos=("inline" if argv[1]=="True" else False),cssclass=name)
formatter = pygments.formatters.HtmlFormatter(linenos=("table" if argv[1]=="True" else False), cssclass=name)
code = argv[2]
result = pygments.highlight(code.replace("\:",":"), lexer, formatter)
css = formatter.get_style_defs(name)
result = pygments.highlight(code.replace("\:",":"), lexer, formatter).replace(".:","&period;:")
css = formatter.get_style_defs()
if argv[1]=="True":
css += "\n."+name+"table {width: 100%; display: block;}"
css += "\n."+name+"table tr {width: 100%; display: block;}"
css += "\n."+name+"table tbody {width: 100%; display: block;}"
css += "\n."+name+"table .linenos {width: 4%; display: inline-block;vertical-align: top;}"
css += "\n."+name+"table .code {width: 95%; display: inline-block;vertical-align: top;}"
css += "\n."+name+"table td pre {vertical-align: top; overflow: hidden; overflow-x: scroll;}"
else:
css += "\n."+name+" pre {overflow: hidden; overflow-x: scroll;}"
if not name+".css" in self._includes.keys():
self._includes[name+".css"] = css
......@@ -61,3 +71,6 @@ class CodeHighlight:
except pygments.util.ClassNotFound:
die(messages.unknown_language.format(argv[0]))
except Exception as e:
print(e)
......@@ -18,6 +18,7 @@
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
import cgi
import markdown
import venc2.l10n
import venc2.helpers
......@@ -25,6 +26,7 @@ from copy import deepcopy
from venc2.helpers import get_formatted_message
from venc2.helpers import highlight_value
from venc2.helpers import notify
from venc2.helpers import remove_by_value
from venc2.helpers import PatternInvalidArgument
from venc2.l10n import messages
......@@ -37,10 +39,13 @@ OPEN_SYMBOL = ".:"
SEPARATOR = "::"
CLOSE_SYMBOL = ":."
markup_language_errors = []
# Because parsing is recursive we want to avoid useless computation
# by splitting a given string and mark where exactly are the patterns
# we want to process.
class PreProcessor():
def __init__(self, string):
self.blacklist = list()
......@@ -78,18 +83,36 @@ class PreProcessor():
# append last substring
self.sub_strings.append(string)
# To avoid markup language collision we split appart Code snippet from the rest of the entry content
self.keep_appart_from_markup_index = list()
def process_markup_language(self, markup_language, source):
new_sub_strings = [""]
for index in range(0, len(self.sub_strings)):
if not index in self.keep_appart_from_markup_index:
new_sub_strings[-1] += self.sub_strings[index]
if index == len(self.sub_strings) - 1:
new_sub_strings[-1] = parse_markup_language(new_sub_strings[-1], markup_language, source)
else:
new_sub_strings[-1] = parse_markup_language(new_sub_strings[-1], markup_language, source)
new_sub_strings.append(self.sub_strings[index])
new_sub_strings.append("")
def merge_batches(batches):
return ''.join(batches.sub_strings)
return ''.join(new_sub_strings)
""" should be replaced by ''.join(sub_string)
def get_final_string(processed):
output = str()
for chunk in processed.sub_strings:
output += chunk
def parse_markup_language(string, markup_language, source):
if markup_language == "Markdown":
string = markdown.markdown(string)
elif markup_language != "none":
err = messages.unknown_markup_language.format(markup_language, source)
if not err in markup_language_errors:
notify(messages.unknown_markup_language.format(markup_language, source), "RED")
markup_language_errors.append(err)
return output
"""
return string
class Processor():
def __init__(self):
......@@ -114,7 +137,7 @@ class Processor():
except UnknownContextual as e:
output = self.handle_error(
messages.unknown_contextual.format(e),
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
error_origin = "{0["+str(e)[1:-1]+']}'
)
......@@ -122,34 +145,34 @@ class Processor():
if str(e)[1:-1] == pattern:
output = self.handle_error(
messages.unknown_pattern.format(pattern),
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
error_origin = [':'+pattern+':']
)
else:
output = self.handle_error(
messages.unknown_contextual.format(e),
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
error_origin = [':'+str(e)[1:-1]+':','{0['+str(e)[1:-1]+']}']
)
except AttributeError as e:
output = self.handle_error(
messages.unknown_contextual.format(e),
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
error_origin = [str(e).split("'")[-2]]
)
except IndexError:
output = self.handle_error(
pattern+": "+messages.not_enough_args,
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
[pattern]
)
except PatternInvalidArgument as e:
output = self.handle_error(
messages.wrong_pattern_argument.format(e.name, e.value, pattern)+' '+e.message,
""+pattern+"§§"+"§§".join(argv)+"§~",
",;"+pattern+";;"+";;".join(argv)+";,",
error_origin = [".:"+pattern+"::"+"::".join(argv)+":."]
)
......@@ -206,7 +229,7 @@ class Processor():
return True
# Process queue
def batch_process(self, input_pre_processed, ressource, markdown=False, escape=False):
def batch_process(self, input_pre_processed, ressource, escape=False):
self.ressource = ressource
pre_processed = deepcopy(input_pre_processed)
if len(pre_processed.patterns_index) == 0:
......@@ -216,6 +239,9 @@ class Processor():
to_remove = list()
for index in pre_processed.patterns_index:
current_pattern = pre_processed.sub_strings[index][2:-2].split('::')[0]
if current_pattern == "CodeHighlight":
pre_processed.keep_appart_from_markup_index.append(index)
if not current_pattern in self.blacklist:
pre_processed.sub_strings[index] = self.process(pre_processed.sub_strings[index], escape)
......@@ -223,19 +249,6 @@ class Processor():
if self.if_not_present(pre_processed.sub_strings[index]):
to_remove.append(index)
if current_pattern in self.clean_after_and_before and markdown:
try:
pre_processed.sub_strings[index-1] = pre_processed.sub_strings[index-1][:-3] # remove <p>
except IndexError:
pass
try:
pre_processed.sub_strings[index+1] = pre_processed.sub_strings[index+1][4:] # remove </p>
except IndexError:
pass
for index in to_remove:
pre_processed.patterns_index = remove_by_value(pre_processed.patterns_index, index)
......@@ -255,32 +268,30 @@ class Processor():
elif string[i:i+2] == CLOSE_SYMBOL:
close_symbol_pos.append(i)
if len(close_symbol_pos) <= len(open_symbol_pos) and len(close_symbol_pos) != 0 and len(open_symbol_pos) != 0:
if open_symbol_pos[-1] < close_symbol_pos[0]:
fields = [field for field in string[open_symbol_pos[-1]+2:close_symbol_pos[0]].split(SEPARATOR) if field != '']
if len(close_symbol_pos) == len(open_symbol_pos) and len(close_symbol_pos) != 0 and len(open_symbol_pos) != 0:
if open_symbol_pos[0] < close_symbol_pos[-1]:
fields = [field for field in string[open_symbol_pos[0]+2:close_symbol_pos[-1]].split(SEPARATOR) if field != '']
if not fields[0] in self.blacklist:
output = self.run_pattern(fields[0], fields[1:])
if escape:
return self.process(
string[:open_symbol_pos[-1]]+
string[:open_symbol_pos[0]]+
cgi.escape(output).encode(
'ascii',
'xmlcharrefreplace'
).decode(
encoding='ascii'
)+
string[close_symbol_pos[0]+2:],
escape=True
string[close_symbol_pos[-1]+2:],
escape
)
else:
return self.process(
string[
:open_symbol_pos[-1]]+
str(output)+
string[close_symbol_pos[0]+2:
],
escape=False
string[:open_symbol_pos[0]]+