Commit e7eb382d authored by Denis Salem's avatar Denis Salem
Browse files

handle forbidden patterns

parent 89c342a9
......@@ -124,10 +124,10 @@
DONE | Add more specific pattern to detect location (IfInCategories,IfInArchives,IfInFirstPage, etc).
DONE | Fix: Import asset from external theme if necessary.
DONE | Export empty blog.
WIP | Allow disabling exportation of specific kind of thread.
DONE | Allow disabling exportation of specific kind of thread.
DONE | Add categories tree in entry.
TODO | Add support for embed content (peertube, youtube, soundcloud, etc)
TODO | Export entry in nicely named folder instead of permalink
TODO | Add categories tree in entry.
TODO | Client-side search engine.
TODO | Fix unit test.
TODO | Remove illegal character from path.
......
......@@ -48,7 +48,7 @@ from venc2.threads.main import MainThread
datastore = DataStore()
code_highlight = CodeHighlight(datastore.blog_configuration["code_highlight_css_override"])
non_contextual_pattern_names_datastore = {
non_contextual_pattern_names_entry = {
# General entry data
"GetEntryTitle" : "get_entry_title",
"GetEntryID" : "get_entry_id",
......@@ -60,8 +60,19 @@ non_contextual_pattern_names_datastore = {
"GetEntryDate" : "get_entry_date",
"GetEntryDateURL" : "get_entry_date_url",
"GetEntryURL" : "get_entry_url",
"ForEntryAuthors" : "for_entry_authors",
"ForEntryTags" : "for_entry_tags",
# General blog data
# Extra metadata getter
"LeavesForEntryCategories" : "leaves_for_entry_categories",
"TreeForEntryCategories" : "tree_for_entry_categories",
"GetEntryMetadata" : "get_entry_metadata",
"GetEntryMetadataIfExists" : "get_entry_metadata_if_exists"
}
non_contextual_pattern_names_entry_keys = non_contextual_pattern_names_entry.keys()
non_contextual_pattern_names_blog = {
"GetAuthorName" : "get_author_name",
"GetBlogName" : "get_blog_name",
"GetBlogDescription" : "get_blog_description",
......@@ -73,16 +84,16 @@ non_contextual_pattern_names_datastore = {
"GetAuthorEmail" : "get_author_email",
# Extra metadata getter
"GetEntryMetadata" : "get_entry_metadata",
"GetEntryMetadataIfExists" : "get_entry_metadata_if_exists",
"GetBlogMetadataIfExists" : "get_blog_metadata_if_exists",
"ForEntryAuthors" : "for_entry_authors",
"ForEntryTags" : "for_entry_tags",
"ForBlogDates" : "for_blog_dates",
"LeafForBlogCategories" : "leaf_for_blog_categories",
"TreeForBlogCategories" : "tree_for_blog_categories"
}
non_contextual_pattern_names_datastore = {
**non_contextual_pattern_names_blog,
**non_contextual_pattern_names_entry
}
non_contextual_pattern_names_ml = {
"CodeHighlight" : code_highlight.highlight,
"Latex2MathML" : Latex2MathML
......@@ -163,6 +174,7 @@ def export_blog(argv=list()):
entry.rss_wrapper.above = PreProcessor(''.join(processor.batch_process(entry.rss_wrapper.above, "rssEntry.html", False).sub_strings))
entry.rss_wrapper.below = PreProcessor(''.join(processor.batch_process(entry.rss_wrapper.below, "rssEntry.html", False).sub_strings))
processor.forbidden = non_contextual_pattern_names_entry_keys
theme.header = PreProcessor(''.join(processor.batch_process(theme.header, "header.html").sub_strings))
theme.footer = PreProcessor(''.join(processor.batch_process(theme.footer, "footer.html").sub_strings))
theme.rssHeader = PreProcessor(''.join(processor.batch_process(theme.rss_header, "rssHeader.html").sub_strings))
......@@ -174,19 +186,19 @@ def export_blog(argv=list()):
# Starting second pass and exporting
thread = MainThread(messages.export_main_thread, datastore, theme, contextual_pattern_names)
thread = MainThread(messages.export_main_thread, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_archives"]:
thread = DatesThread(messages.export_archives, datastore, theme, contextual_pattern_names)
thread = DatesThread(messages.export_archives, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_categories"]:
thread = CategoriesThread(messages.export_categories, datastore, theme, contextual_pattern_names)
thread = CategoriesThread(messages.export_categories, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_single_entries"]:
thread = EntriesThread(messages.export_single_entries, datastore, theme, contextual_pattern_names)
thread = EntriesThread(messages.export_single_entries, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
# Copy assets and extra files
......
......@@ -50,6 +50,7 @@ class DataStore:
else:
self.entries.append(Entry(filename))
''' Update entriesPerDates '''
if self.blog_configuration["path"]["dates_directory_name"] != '':
formatted_date = self.entries[-1].date.strftime(self.blog_configuration["path"]["dates_directory_name"])
......@@ -196,11 +197,12 @@ class DataStore:
dates = [o for o in self.blog_dates if o["date"] not in self.disable_threads]
return merge(dates, argv)
def perform_recursion(self, opening_node, opening_branch, closing_branch, closing_node, tree):
def build_html_categories_tree(self, opening_node, opening_branch, closing_branch, closing_node, tree):
output_string = opening_node
for node in sorted(tree, key = lambda x : x.value):
if node.value in self.disable_threads:
continue
variables = {
"item" : node.value,
"count" : node.count,
......@@ -212,7 +214,7 @@ class DataStore:
output_string += opening_branch.format(**variables) + closing_branch.format(**variables)
else:
output_string += opening_branch.format(**variables) + self.perform_recursion(
output_string += opening_branch.format(**variables) + self.build_html_categories_tree(
opening_node,
opening_branch,
closing_branch,
......@@ -225,10 +227,23 @@ class DataStore:
return output_string + closing_node
def tree_for_entry_categories(self, argv):
# compute once categories tree and deliver baked html
if self.entries[self.requested_entry_index].html_categories_tree == None:
self.entries[self.requested_entry_index].html_categories_tree = self.build_html_categories_tree(
argv[0], #opening_node
argv[1], #opening_branch
argv[2], #closing_branch
argv[3], #closing_node
self.entries[self.requested_entry_index].categories_tree
)
return self.entries[self.requested_entry_index].html_categories_tree
def tree_for_blog_categories(self, argv):
# compute once categories tree and deliver baked html
if self.html_categories_tree == None:
self.html_categories_tree = self.perform_recursion(
self.html_categories_tree = self.build_html_categories_tree(
argv[0], #opening_node
argv[1], #opening_branch
argv[2], #closing_branch
......@@ -244,9 +259,8 @@ class DataStore:
def for_entry_authors(self, argv):
return merge(self.entries[self.requested_entry_index].authors, argv)
def leaf_for_blog_categories(self, argv):
output_string = str()
return output_string
def leaves_for_entry_categories(self, argv):
return merge(self.entries[self.requested_entry_index].categories_leaves, argv)
......
......@@ -121,13 +121,13 @@ class Entry:
for category in self.raw_categories:
category_leaf = category.split(' > ')[-1].strip()
if len(category_leaf) != 0:
category_leaf_url = str()
category_leaf_url = ".:GetRelativeOrigin:."
for sub_category in category.split(' > '):
category_leaf_url +=sub_category.strip()+'/'
self.categories_leaves.append({
"categoryLeaf": category_leaf,
"categoryLeafPath":category_leaf_url
"item": category_leaf,
"path":category_leaf_url
})
except IndexError : # when list is empty
......@@ -135,6 +135,7 @@ class Entry:
self.categories_tree = []
build_categories_tree(-1, self.raw_categories, self.categories_tree, -1)
self.html_categories_tree = None
''' Iterate through entries folder '''
def yield_entries_content():
......@@ -169,52 +170,9 @@ def yield_entries_content():
except FileNotFoundError:
die(messages.file_not_found.format(os.getcwd()+"/entries"))
''' User for set the id of new entry '''
def get_latest_entryID():
entries_list = sorted(yield_entries_content(), key = lambda entry : int(entry.split("__")[0]))
if len(entries_list) != 0:
return int(entries_list[-1].split("__")[0])
else:
return 0
'''
def GetEntriesPerCategories(entries):
entriesPerCategories = list()
for entry in entries.keys():
stream = entries[entry].split("---\n")[0]
try:
data = yaml.load(stream)
except yaml.scanner.ScannerError:
Die(Messages.possibleMalformedEntry.format(entry))
except yaml.parser.ParserError:
Die(Messages.possibleMalformedEntry.format(entry))
if data != None:
try:
for category in data["categories"].split(","):
if category != '':
nodes = entriesPerCategories
path = str()
for subCategory in category.split(" > "):
path += subCategory+'/'
try:
selectedKey = GetKeyByName(nodes, subCategory.strip())
selectedKey.relatedTo.append(entry)
selectedKey.count += 1
selectedKey.relatedTo = list(set(selectedKey.relatedTo))
except Exception as e:
selectedKey = Metadata(subCategory.strip(), entry)
selectedKey.path = path
nodes.append(selectedKey)
nodes = selectedKey.childs
except TypeError:
Die(Messages.possibleMalformedEntry.format(entry))
return entriesPerCategories
'''
......@@ -28,6 +28,23 @@ class MetadataNode:
self.related_to = [entry_index]
self.childs = list()
def build_categories_leaves():
try:
for category in self.raw_categories:
category_leaf = category.split(' > ')[-1].strip()
if len(category_leaf) != 0:
category_leaf_url = str()
for sub_category in category.split(' > '):
category_leaf_url +=sub_category.strip()+'/'
self.categories_leaves.append({
"categoryLeaf": category_leaf,
"categoryLeafPath":category_leaf_url
})
except IndexError : # when list is empty
pass
def build_categories_tree(entry_index, input_list, output_tree, max_weight, set_max_weight=None):
for category in input_list:
branch = category.split(' > ')
......
......@@ -52,7 +52,7 @@ class Theme:
source = ""
for ext in argv[1].split(','):
# Set media once, and get complete path later.
source += str("<source src=\".:GetRelativeOrigin:.{0}.{1}\" type=\""+media_type+"/{1}\">\n").format(argv[0].strip(), ext.strip())
source += str("<source src=\"{0}.{1}\" type=\""+media_type+"/{1}\">\n").format(argv[0].strip(), ext.strip())
f = {}
f["source"] = source
......
......@@ -86,3 +86,4 @@ class Messages:
unknown_markup_language = "{0}: Unsupported markup language."
tex_math_error = "Malformed input string or unsupported markup from latex2mathml."
missing_separator_in_entry = "Missing separator {0}."
pattern_is_forbidden_here = "The pattern '{0}' is forbidden here."
......@@ -84,5 +84,6 @@ class Messages:
unknown_markup_language = "{0}: Langage de balisage non supporté dans {1}."
tex_math_error = "Chaine mal formé ou balise non supporté par latex2mathml."
missing_separator_in_entry = "Il manque le séparateur {0}."
pattern_is_forbidden_here = "Le motif '{0}' n'est pas autorisé ici."
......@@ -144,6 +144,7 @@ def parse_markup_language(string, markup_language, source):
class Processor():
def __init__(self):
self.forbidden = []
self.functions = dict()
self.currentString = str()
self.ressource = str()
......@@ -273,10 +274,18 @@ class Processor():
to_remove = list()
for index in pre_processed.patterns_index:
current_pattern = pre_processed.sub_strings[index][2:-2].split('::')[0]
if current_pattern in ["CodeHighlight", "Latex2MathML","IncludeFile","audio","video"]:
if current_pattern in ["CodeHighlight", "Latex2MathML", "IncludeFile", "audio", "video"]:
pre_processed.keep_appart_from_markup_index.append(index)
if not current_pattern in self.blacklist:
""" this is tested twice??? """
if current_pattern in self.forbidden:
self.handle_error(
messages.pattern_is_forbidden_here.format(current_pattern),
",;"+current_pattern+";;"+";;".join(pre_processed.sub_strings[index][2:-2])+";,",
error_origin = [current_pattern]
)
if (not current_pattern in self.blacklist) and (not current_pattern in self.forbidden) :
pre_processed.sub_strings[index] = self.process(pre_processed.sub_strings[index], escape)
# check if there is residual patterns
......@@ -305,7 +314,8 @@ class Processor():
if len(close_symbol_pos) <= len(open_symbol_pos) and len(close_symbol_pos) != 0 and len(open_symbol_pos) != 0:
if open_symbol_pos[-1] < close_symbol_pos[-1]:
fields = [field for field in string[open_symbol_pos[-1]+2:close_symbol_pos[-1]].split(SEPARATOR) if field != '']
if not fields[0] in self.blacklist:
""" this is tested twice??? """
if not fields[0] in self.blacklist and (not fields[0] in self.forbidden):
output = self.run_pattern(fields[0], fields[1:])
if escape:
return self.process(
......
......@@ -27,9 +27,10 @@ from venc2.l10n import messages
from venc2.patterns.processor import Processor
class Thread:
def __init__(self, prompt, datastore, theme, patterns):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
# Notify wich thread is processed
notify(prompt)
self.forbidden = forbidden
self.entries_per_page = int(datastore.blog_configuration["entries_per_pages"])
self.disable_threads = datastore.disable_threads
......@@ -88,7 +89,11 @@ class Thread:
def get_previous_page(self, argv=list()):
if self.current_page > 0:
destination_page_number = str(self.current_page - 1)
previous_entry_id = self.current_entry.previous_entry.id
try:
previous_entry_id = self.current_entry.previous_entry.id
except AttributeError:
previous_entry_id = -1
if self.current_page == 1:
filename = self.filename.format(**{"page_number" : "", "entry_id" : previous_entry_id})
......@@ -188,7 +193,9 @@ class Thread:
else:
for page in self.pages:
self.processor.forbidden = self.forbidden
output = ''.join(self.processor.batch_process(self.theme.header, "header.html").sub_strings)
self.processor.forbidden = []
columns_number = self.datastore.blog_configuration["columns"]
columns_counter = 0
......@@ -215,6 +222,7 @@ class Thread:
for column in columns:
output += '<div id="__VENC_COLUMN_'+str(columns_counter)+'__" class="__VENC_COLUMN__">'+column+'</div>'
self.processor.forbidden = self.forbidden
output += ''.join(self.processor.batch_process(self.theme.footer, "footer.html").sub_strings)
if self.in_thread:
......
......@@ -23,8 +23,8 @@ from venc2.helpers import notify
from venc2.threads import Thread
class CategoriesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns):
super().__init__(prompt, datastore, theme, patterns)
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
self.filename = self.datastore.blog_configuration["path"]["index_file_name"]
self.relative_origin = ""
......
......@@ -23,8 +23,8 @@ from venc2.helpers import notify
from venc2.threads import Thread
class DatesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns):
super().__init__(prompt, datastore, theme, patterns)
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
self.filename = self.datastore.blog_configuration["path"]["index_file_name"]
self.relative_origin = "../"
......
......@@ -23,8 +23,8 @@ from venc2.helpers import notify
from venc2.threads import Thread
class EntriesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns):
super().__init__(prompt, datastore, theme, patterns)
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
self.entries_per_page = 1 #override value
self.organize_entries([
entry for entry in datastore.get_entries(
......
......@@ -20,8 +20,8 @@
from venc2.threads import Thread
class MainThread(Thread):
def __init__(self, prompt, datastore, theme, patterns):
super().__init__(prompt, datastore, theme, patterns)
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
if datastore.blog_configuration["disable_main_thread"]:
self.pages_count == 0
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment