Skip to content
GitLab
Menu
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Denis Salem
VenC
Commits
e7eb382d
Commit
e7eb382d
authored
Nov 05, 2018
by
Denis Salem
Browse files
handle forbidden patterns
parent
89c342a9
Changes
14
Hide whitespace changes
Inline
Side-by-side
changelog.md
View file @
e7eb382d
...
...
@@ -124,10 +124,10 @@
DONE | Add more specific pattern to detect location (IfInCategories,IfInArchives,IfInFirstPage, etc).
DONE | Fix: Import asset from external theme if necessary.
DONE | Export empty blog.
WIP | Allow disabling exportation of specific kind of thread.
DONE | Allow disabling exportation of specific kind of thread.
DONE | Add categories tree in entry.
TODO | Add support for embed content (peertube, youtube, soundcloud, etc)
TODO | Export entry in nicely named folder instead of permalink
TODO | Add categories tree in entry.
TODO | Client-side search engine.
TODO | Fix unit test.
TODO | Remove illegal character from path.
...
...
src/venc2/commands/export.py
View file @
e7eb382d
...
...
@@ -48,7 +48,7 @@ from venc2.threads.main import MainThread
datastore
=
DataStore
()
code_highlight
=
CodeHighlight
(
datastore
.
blog_configuration
[
"code_highlight_css_override"
])
non_contextual_pattern_names_
datastore
=
{
non_contextual_pattern_names_
entry
=
{
# General entry data
"GetEntryTitle"
:
"get_entry_title"
,
"GetEntryID"
:
"get_entry_id"
,
...
...
@@ -60,8 +60,19 @@ non_contextual_pattern_names_datastore = {
"GetEntryDate"
:
"get_entry_date"
,
"GetEntryDateURL"
:
"get_entry_date_url"
,
"GetEntryURL"
:
"get_entry_url"
,
"ForEntryAuthors"
:
"for_entry_authors"
,
"ForEntryTags"
:
"for_entry_tags"
,
# General blog data
# Extra metadata getter
"LeavesForEntryCategories"
:
"leaves_for_entry_categories"
,
"TreeForEntryCategories"
:
"tree_for_entry_categories"
,
"GetEntryMetadata"
:
"get_entry_metadata"
,
"GetEntryMetadataIfExists"
:
"get_entry_metadata_if_exists"
}
non_contextual_pattern_names_entry_keys
=
non_contextual_pattern_names_entry
.
keys
()
non_contextual_pattern_names_blog
=
{
"GetAuthorName"
:
"get_author_name"
,
"GetBlogName"
:
"get_blog_name"
,
"GetBlogDescription"
:
"get_blog_description"
,
...
...
@@ -73,16 +84,16 @@ non_contextual_pattern_names_datastore = {
"GetAuthorEmail"
:
"get_author_email"
,
# Extra metadata getter
"GetEntryMetadata"
:
"get_entry_metadata"
,
"GetEntryMetadataIfExists"
:
"get_entry_metadata_if_exists"
,
"GetBlogMetadataIfExists"
:
"get_blog_metadata_if_exists"
,
"ForEntryAuthors"
:
"for_entry_authors"
,
"ForEntryTags"
:
"for_entry_tags"
,
"ForBlogDates"
:
"for_blog_dates"
,
"LeafForBlogCategories"
:
"leaf_for_blog_categories"
,
"TreeForBlogCategories"
:
"tree_for_blog_categories"
}
non_contextual_pattern_names_datastore
=
{
**
non_contextual_pattern_names_blog
,
**
non_contextual_pattern_names_entry
}
non_contextual_pattern_names_ml
=
{
"CodeHighlight"
:
code_highlight
.
highlight
,
"Latex2MathML"
:
Latex2MathML
...
...
@@ -163,6 +174,7 @@ def export_blog(argv=list()):
entry
.
rss_wrapper
.
above
=
PreProcessor
(
''
.
join
(
processor
.
batch_process
(
entry
.
rss_wrapper
.
above
,
"rssEntry.html"
,
False
).
sub_strings
))
entry
.
rss_wrapper
.
below
=
PreProcessor
(
''
.
join
(
processor
.
batch_process
(
entry
.
rss_wrapper
.
below
,
"rssEntry.html"
,
False
).
sub_strings
))
processor
.
forbidden
=
non_contextual_pattern_names_entry_keys
theme
.
header
=
PreProcessor
(
''
.
join
(
processor
.
batch_process
(
theme
.
header
,
"header.html"
).
sub_strings
))
theme
.
footer
=
PreProcessor
(
''
.
join
(
processor
.
batch_process
(
theme
.
footer
,
"footer.html"
).
sub_strings
))
theme
.
rssHeader
=
PreProcessor
(
''
.
join
(
processor
.
batch_process
(
theme
.
rss_header
,
"rssHeader.html"
).
sub_strings
))
...
...
@@ -174,19 +186,19 @@ def export_blog(argv=list()):
# Starting second pass and exporting
thread
=
MainThread
(
messages
.
export_main_thread
,
datastore
,
theme
,
contextual_pattern_names
)
thread
=
MainThread
(
messages
.
export_main_thread
,
datastore
,
theme
,
contextual_pattern_names
,
non_contextual_pattern_names_entry_keys
)
thread
.
do
()
if
not
datastore
.
blog_configuration
[
"disable_archives"
]:
thread
=
DatesThread
(
messages
.
export_archives
,
datastore
,
theme
,
contextual_pattern_names
)
thread
=
DatesThread
(
messages
.
export_archives
,
datastore
,
theme
,
contextual_pattern_names
,
non_contextual_pattern_names_entry_keys
)
thread
.
do
()
if
not
datastore
.
blog_configuration
[
"disable_categories"
]:
thread
=
CategoriesThread
(
messages
.
export_categories
,
datastore
,
theme
,
contextual_pattern_names
)
thread
=
CategoriesThread
(
messages
.
export_categories
,
datastore
,
theme
,
contextual_pattern_names
,
non_contextual_pattern_names_entry_keys
)
thread
.
do
()
if
not
datastore
.
blog_configuration
[
"disable_single_entries"
]:
thread
=
EntriesThread
(
messages
.
export_single_entries
,
datastore
,
theme
,
contextual_pattern_names
)
thread
=
EntriesThread
(
messages
.
export_single_entries
,
datastore
,
theme
,
contextual_pattern_names
,
non_contextual_pattern_names_entry_keys
)
thread
.
do
()
# Copy assets and extra files
...
...
src/venc2/datastore/datastore.py
View file @
e7eb382d
...
...
@@ -50,6 +50,7 @@ class DataStore:
else
:
self
.
entries
.
append
(
Entry
(
filename
))
''' Update entriesPerDates '''
if
self
.
blog_configuration
[
"path"
][
"dates_directory_name"
]
!=
''
:
formatted_date
=
self
.
entries
[
-
1
].
date
.
strftime
(
self
.
blog_configuration
[
"path"
][
"dates_directory_name"
])
...
...
@@ -196,11 +197,12 @@ class DataStore:
dates
=
[
o
for
o
in
self
.
blog_dates
if
o
[
"date"
]
not
in
self
.
disable_threads
]
return
merge
(
dates
,
argv
)
def
perform_recursion
(
self
,
opening_node
,
opening_branch
,
closing_branch
,
closing_node
,
tree
):
def
build_html_categories_tree
(
self
,
opening_node
,
opening_branch
,
closing_branch
,
closing_node
,
tree
):
output_string
=
opening_node
for
node
in
sorted
(
tree
,
key
=
lambda
x
:
x
.
value
):
if
node
.
value
in
self
.
disable_threads
:
continue
variables
=
{
"item"
:
node
.
value
,
"count"
:
node
.
count
,
...
...
@@ -212,7 +214,7 @@ class DataStore:
output_string
+=
opening_branch
.
format
(
**
variables
)
+
closing_branch
.
format
(
**
variables
)
else
:
output_string
+=
opening_branch
.
format
(
**
variables
)
+
self
.
perform_recursion
(
output_string
+=
opening_branch
.
format
(
**
variables
)
+
self
.
build_html_categories_tree
(
opening_node
,
opening_branch
,
closing_branch
,
...
...
@@ -225,10 +227,23 @@ class DataStore:
return
output_string
+
closing_node
def
tree_for_entry_categories
(
self
,
argv
):
# compute once categories tree and deliver baked html
if
self
.
entries
[
self
.
requested_entry_index
].
html_categories_tree
==
None
:
self
.
entries
[
self
.
requested_entry_index
].
html_categories_tree
=
self
.
build_html_categories_tree
(
argv
[
0
],
#opening_node
argv
[
1
],
#opening_branch
argv
[
2
],
#closing_branch
argv
[
3
],
#closing_node
self
.
entries
[
self
.
requested_entry_index
].
categories_tree
)
return
self
.
entries
[
self
.
requested_entry_index
].
html_categories_tree
def
tree_for_blog_categories
(
self
,
argv
):
# compute once categories tree and deliver baked html
if
self
.
html_categories_tree
==
None
:
self
.
html_categories_tree
=
self
.
perform_recursion
(
self
.
html_categories_tree
=
self
.
build_html_categories_tree
(
argv
[
0
],
#opening_node
argv
[
1
],
#opening_branch
argv
[
2
],
#closing_branch
...
...
@@ -244,9 +259,8 @@ class DataStore:
def
for_entry_authors
(
self
,
argv
):
return
merge
(
self
.
entries
[
self
.
requested_entry_index
].
authors
,
argv
)
def
leaf_for_blog_categories
(
self
,
argv
):
output_string
=
str
()
return
output_string
def
leaves_for_entry_categories
(
self
,
argv
):
return
merge
(
self
.
entries
[
self
.
requested_entry_index
].
categories_leaves
,
argv
)
...
...
src/venc2/datastore/entry.py
View file @
e7eb382d
...
...
@@ -121,13 +121,13 @@ class Entry:
for
category
in
self
.
raw_categories
:
category_leaf
=
category
.
split
(
' > '
)[
-
1
].
strip
()
if
len
(
category_leaf
)
!=
0
:
category_leaf_url
=
str
()
category_leaf_url
=
".:GetRelativeOrigin:."
for
sub_category
in
category
.
split
(
' > '
):
category_leaf_url
+=
sub_category
.
strip
()
+
'/'
self
.
categories_leaves
.
append
({
"
categoryLeaf
"
:
category_leaf
,
"
categoryLeafP
ath"
:
category_leaf_url
"
item
"
:
category_leaf
,
"
p
ath"
:
category_leaf_url
})
except
IndexError
:
# when list is empty
...
...
@@ -135,6 +135,7 @@ class Entry:
self
.
categories_tree
=
[]
build_categories_tree
(
-
1
,
self
.
raw_categories
,
self
.
categories_tree
,
-
1
)
self
.
html_categories_tree
=
None
''' Iterate through entries folder '''
def
yield_entries_content
():
...
...
@@ -169,52 +170,9 @@ def yield_entries_content():
except
FileNotFoundError
:
die
(
messages
.
file_not_found
.
format
(
os
.
getcwd
()
+
"/entries"
))
''' User for set the id of new entry '''
def
get_latest_entryID
():
entries_list
=
sorted
(
yield_entries_content
(),
key
=
lambda
entry
:
int
(
entry
.
split
(
"__"
)[
0
]))
if
len
(
entries_list
)
!=
0
:
return
int
(
entries_list
[
-
1
].
split
(
"__"
)[
0
])
else
:
return
0
'''
def GetEntriesPerCategories(entries):
entriesPerCategories = list()
for entry in entries.keys():
stream = entries[entry].split("---
\n
")[0]
try:
data = yaml.load(stream)
except yaml.scanner.ScannerError:
Die(Messages.possibleMalformedEntry.format(entry))
except yaml.parser.ParserError:
Die(Messages.possibleMalformedEntry.format(entry))
if data != None:
try:
for category in data["categories"].split(","):
if category != '':
nodes = entriesPerCategories
path = str()
for subCategory in category.split(" > "):
path += subCategory+'/'
try:
selectedKey = GetKeyByName(nodes, subCategory.strip())
selectedKey.relatedTo.append(entry)
selectedKey.count += 1
selectedKey.relatedTo = list(set(selectedKey.relatedTo))
except Exception as e:
selectedKey = Metadata(subCategory.strip(), entry)
selectedKey.path = path
nodes.append(selectedKey)
nodes = selectedKey.childs
except TypeError:
Die(Messages.possibleMalformedEntry.format(entry))
return entriesPerCategories
'''
src/venc2/datastore/metadata.py
View file @
e7eb382d
...
...
@@ -28,6 +28,23 @@ class MetadataNode:
self
.
related_to
=
[
entry_index
]
self
.
childs
=
list
()
def
build_categories_leaves
():
try
:
for
category
in
self
.
raw_categories
:
category_leaf
=
category
.
split
(
' > '
)[
-
1
].
strip
()
if
len
(
category_leaf
)
!=
0
:
category_leaf_url
=
str
()
for
sub_category
in
category
.
split
(
' > '
):
category_leaf_url
+=
sub_category
.
strip
()
+
'/'
self
.
categories_leaves
.
append
({
"categoryLeaf"
:
category_leaf
,
"categoryLeafPath"
:
category_leaf_url
})
except
IndexError
:
# when list is empty
pass
def
build_categories_tree
(
entry_index
,
input_list
,
output_tree
,
max_weight
,
set_max_weight
=
None
):
for
category
in
input_list
:
branch
=
category
.
split
(
' > '
)
...
...
src/venc2/datastore/theme.py
View file @
e7eb382d
...
...
@@ -52,7 +52,7 @@ class Theme:
source
=
""
for
ext
in
argv
[
1
].
split
(
','
):
# Set media once, and get complete path later.
source
+=
str
(
"<source src=
\"
.:GetRelativeOrigin:.
{0}.{1}
\"
type=
\"
"
+
media_type
+
"/{1}
\"
>
\n
"
).
format
(
argv
[
0
].
strip
(),
ext
.
strip
())
source
+=
str
(
"<source src=
\"
{0}.{1}
\"
type=
\"
"
+
media_type
+
"/{1}
\"
>
\n
"
).
format
(
argv
[
0
].
strip
(),
ext
.
strip
())
f
=
{}
f
[
"source"
]
=
source
...
...
src/venc2/l10n/en.py
View file @
e7eb382d
...
...
@@ -86,3 +86,4 @@ class Messages:
unknown_markup_language
=
"{0}: Unsupported markup language."
tex_math_error
=
"Malformed input string or unsupported markup from latex2mathml."
missing_separator_in_entry
=
"Missing separator {0}."
pattern_is_forbidden_here
=
"The pattern '{0}' is forbidden here."
src/venc2/l10n/fr.py
View file @
e7eb382d
...
...
@@ -84,5 +84,6 @@ class Messages:
unknown_markup_language
=
"{0}: Langage de balisage non supporté dans {1}."
tex_math_error
=
"Chaine mal formé ou balise non supporté par latex2mathml."
missing_separator_in_entry
=
"Il manque le séparateur {0}."
pattern_is_forbidden_here
=
"Le motif '{0}' n'est pas autorisé ici."
src/venc2/patterns/processor.py
View file @
e7eb382d
...
...
@@ -144,6 +144,7 @@ def parse_markup_language(string, markup_language, source):
class
Processor
():
def
__init__
(
self
):
self
.
forbidden
=
[]
self
.
functions
=
dict
()
self
.
currentString
=
str
()
self
.
ressource
=
str
()
...
...
@@ -273,10 +274,18 @@ class Processor():
to_remove
=
list
()
for
index
in
pre_processed
.
patterns_index
:
current_pattern
=
pre_processed
.
sub_strings
[
index
][
2
:
-
2
].
split
(
'::'
)[
0
]
if
current_pattern
in
[
"CodeHighlight"
,
"Latex2MathML"
,
"IncludeFile"
,
"audio"
,
"video"
]:
if
current_pattern
in
[
"CodeHighlight"
,
"Latex2MathML"
,
"IncludeFile"
,
"audio"
,
"video"
]:
pre_processed
.
keep_appart_from_markup_index
.
append
(
index
)
if
not
current_pattern
in
self
.
blacklist
:
""" this is tested twice??? """
if
current_pattern
in
self
.
forbidden
:
self
.
handle_error
(
messages
.
pattern_is_forbidden_here
.
format
(
current_pattern
),
",;"
+
current_pattern
+
";;"
+
";;"
.
join
(
pre_processed
.
sub_strings
[
index
][
2
:
-
2
])
+
";,"
,
error_origin
=
[
current_pattern
]
)
if
(
not
current_pattern
in
self
.
blacklist
)
and
(
not
current_pattern
in
self
.
forbidden
)
:
pre_processed
.
sub_strings
[
index
]
=
self
.
process
(
pre_processed
.
sub_strings
[
index
],
escape
)
# check if there is residual patterns
...
...
@@ -305,7 +314,8 @@ class Processor():
if
len
(
close_symbol_pos
)
<=
len
(
open_symbol_pos
)
and
len
(
close_symbol_pos
)
!=
0
and
len
(
open_symbol_pos
)
!=
0
:
if
open_symbol_pos
[
-
1
]
<
close_symbol_pos
[
-
1
]:
fields
=
[
field
for
field
in
string
[
open_symbol_pos
[
-
1
]
+
2
:
close_symbol_pos
[
-
1
]].
split
(
SEPARATOR
)
if
field
!=
''
]
if
not
fields
[
0
]
in
self
.
blacklist
:
""" this is tested twice??? """
if
not
fields
[
0
]
in
self
.
blacklist
and
(
not
fields
[
0
]
in
self
.
forbidden
):
output
=
self
.
run_pattern
(
fields
[
0
],
fields
[
1
:])
if
escape
:
return
self
.
process
(
...
...
src/venc2/threads/__init__.py
View file @
e7eb382d
...
...
@@ -27,9 +27,10 @@ from venc2.l10n import messages
from
venc2.patterns.processor
import
Processor
class
Thread
:
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
# Notify wich thread is processed
notify
(
prompt
)
self
.
forbidden
=
forbidden
self
.
entries_per_page
=
int
(
datastore
.
blog_configuration
[
"entries_per_pages"
])
self
.
disable_threads
=
datastore
.
disable_threads
...
...
@@ -88,7 +89,11 @@ class Thread:
def
get_previous_page
(
self
,
argv
=
list
()):
if
self
.
current_page
>
0
:
destination_page_number
=
str
(
self
.
current_page
-
1
)
previous_entry_id
=
self
.
current_entry
.
previous_entry
.
id
try
:
previous_entry_id
=
self
.
current_entry
.
previous_entry
.
id
except
AttributeError
:
previous_entry_id
=
-
1
if
self
.
current_page
==
1
:
filename
=
self
.
filename
.
format
(
**
{
"page_number"
:
""
,
"entry_id"
:
previous_entry_id
})
...
...
@@ -188,7 +193,9 @@ class Thread:
else
:
for
page
in
self
.
pages
:
self
.
processor
.
forbidden
=
self
.
forbidden
output
=
''
.
join
(
self
.
processor
.
batch_process
(
self
.
theme
.
header
,
"header.html"
).
sub_strings
)
self
.
processor
.
forbidden
=
[]
columns_number
=
self
.
datastore
.
blog_configuration
[
"columns"
]
columns_counter
=
0
...
...
@@ -215,6 +222,7 @@ class Thread:
for
column
in
columns
:
output
+=
'<div id="__VENC_COLUMN_'
+
str
(
columns_counter
)
+
'__" class="__VENC_COLUMN__">'
+
column
+
'</div>'
self
.
processor
.
forbidden
=
self
.
forbidden
output
+=
''
.
join
(
self
.
processor
.
batch_process
(
self
.
theme
.
footer
,
"footer.html"
).
sub_strings
)
if
self
.
in_thread
:
...
...
src/venc2/threads/categories.py
View file @
e7eb382d
...
...
@@ -23,8 +23,8 @@ from venc2.helpers import notify
from
venc2.threads
import
Thread
class
CategoriesThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
)
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
,
forbidden
)
self
.
filename
=
self
.
datastore
.
blog_configuration
[
"path"
][
"index_file_name"
]
self
.
relative_origin
=
""
...
...
src/venc2/threads/dates.py
View file @
e7eb382d
...
...
@@ -23,8 +23,8 @@ from venc2.helpers import notify
from
venc2.threads
import
Thread
class
DatesThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
)
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
,
forbidden
)
self
.
filename
=
self
.
datastore
.
blog_configuration
[
"path"
][
"index_file_name"
]
self
.
relative_origin
=
"../"
...
...
src/venc2/threads/entries.py
View file @
e7eb382d
...
...
@@ -23,8 +23,8 @@ from venc2.helpers import notify
from
venc2.threads
import
Thread
class
EntriesThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
)
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
,
forbidden
)
self
.
entries_per_page
=
1
#override value
self
.
organize_entries
([
entry
for
entry
in
datastore
.
get_entries
(
...
...
src/venc2/threads/main.py
View file @
e7eb382d
...
...
@@ -20,8 +20,8 @@
from
venc2.threads
import
Thread
class
MainThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
)
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
,
forbidden
)
if
datastore
.
blog_configuration
[
"disable_main_thread"
]:
self
.
pages_count
==
0
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment