Commit 74771380 authored by Denis Salem's avatar Denis Salem
Browse files

migration script should be done, catch unicode errors

parent 18fddfe7
......@@ -16,20 +16,20 @@ markup_language: 'Markdown'
path:
category_directory_name: '{category}'
dates_directory_name: '%m-%Y'
entry_file_name: index.html
entry_file_name: indexé.html
ftp: Chemin absolu de votre blog sur l'hôte FTP.
index_file_name: index{page_number}.html
rss_file_name: feed.xml
rss_file_name: feedé.xml
dates_sub_folders: ""
categories_sub_folders: ""
entries_sub_folders: "{entry_title}"
reverse_thread_order: false
rss_thread_lenght: 5
text_editor: gvim
disable_main_thread: True
disable_main_thread: true
disable_threads: ""
disable_archives: True
disable_archives: true
disable_categories: false
disable_single_entries: false
banner: 'https://download.tuxfamily.org/dsalem/img/2017_-_Denis_Salem_-_CC_By_SA_-_VenC-logo.svg'
path_encoding: 'utf-8'
path_encoding: 'windows-1252'
@font-face
{font-family: FiraLight;
src: url(/FiraSans-Light.otf);}
body
{background-image: url('bg.png');
font-family: sans;
font-family: FiraLight;
font-weight: lighter;
color: #AAA;
text-shadow: 0px 2px #000;
background-color: #080808;}
background-color: #181818;}
aside
{display: inline-block;
......@@ -31,7 +35,7 @@ aside ul
blockquote,
.venc_source_Yaml
{background-color: #000;
{background-color: rgba(0,0,0,0.5);
color: #FFF;
font-size: 0.8em;
padding:5px;
......@@ -57,7 +61,7 @@ blockquote p
#content
{display: block;
background-image: url('bg.png');
background-color: #000;
background-color: #101010;
border-left: 1px solid #000;
box-shadow: 0px 0px 10px #000 inset;
position: absolute;
......
......@@ -6,6 +6,8 @@
<meta keywords=".:GetBlogKeywords:." />
<meta description=".:GetBlogDescription:." />
<title>.:GetBlogName:.</title>
<style type="text/css">
</style>
.:GetStyleSheets:.
<link rel="stylesheet" href=".:GetRelativeOrigin:.style.css" type="text/css">
<link rel="alternate" type="application/rss+xml" title=".:GetBlogName:." href="./feed.xml" />
......@@ -16,6 +18,6 @@
<header>
<h1><a href=".:GetRootPage:."><img src=".:GetRelativeOrigin:.logo.png" alt=".:GetBlogName:." title=".:GetBlogName:." /></a></h1>
</header>
.:TreeForBlogCategories::<ul>::<li><a href="{path}#entry-content">{item}</a>::</li>::</ul>:.
.:TreeForBlogCategories::<ul>::<li><a href="{path}#entry-content">{value}</a>::</li>::</ul>:.
</aside>
<div id="content">
......@@ -97,7 +97,7 @@
DONE | More verbosity about wrong pattern arguments.
DONE | More verbosity about malformed entry.
DONE | Clear white spaces in error messages.
DONE | Speed up blog exportation ( almost seven times faster!).
DONE | Speed up blog exportation (almost four times faster!).
DONE | Implement RelativeLocation.
DONE | Optimisation of categories tree.
DONE | Code should be a little bit more PEP 8 compliant.
......@@ -132,20 +132,23 @@
DONE | Add GetRootPage.
DONE | Add SetColor.
DONE | Split install and blog creation chapter in tutorial.
DONE | Fix prevent crash from invalid entry id.
DONE | Support for migration.
DONE | Catch UnicodeEncodeError.
WIP | Serv page.
TODO | Support ForPage in entries thread.
TODO | Blacklist user defined patterns from entry.
TODO | Add support For Atom feed.
TODO | Serv page.
TODO | Open with navigator manual.
TODO | Client-side search engine.
TODO | Fix unit test.
TODO | Reorganize themes dependencies in setup (adding default templates, scripts, and pages to include).
TODO | Add relevant templates for themes.
WIP | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
WIP | fix FAQ latex2mathml is incomplete.
WIP | fix FAQ categories separator is ' > ', not '>'
WIP | fix FAQ subfolders must not start with '/'
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
WIP | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
WIP | fix FAQ latex2mathml is incomplete.
WIP | fix FAQ categories separator is ' > ', not '>'
WIP | fix FAQ subfolders must not start with '/'
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
# version 2.x.x
......
......@@ -20,13 +20,14 @@
import os
import yaml
import codecs
import shutil
from venc2.helpers import die
from venc2.helpers import notify
from venc2.datastore.entry import yield_entries_content
from venc2.l10n import messages
def ReplacePatterns(string):
def replace_patterns(string):
string = string.replace(".:Get::EntryID:.",".:GetEntryID:.")
string = string.replace(".:Get::EntryName:.",".:GetEntryTitle:.")
string = string.replace(".:Get::EntryMonth:.",".:GetEntryMonth:.")
......@@ -47,6 +48,7 @@ def ReplacePatterns(string):
string = string.replace(".:Get::AuthorEmail:.",".:GetAuthorEmail:.")
string = string.replace(".:Get::RelativeOrigin:.",".:GetRelativeOrigin:.")
string = string.replace(".:Get::RelativeLocation:.",".:GetRelativeLocation:.")
string = string.replace(".:RecursiveFor::BlogCategories::",".:TreeForBlogCategories::")
string = string.replace(".:Get::EntryUrl:.",".:GetEntryURL:.")
string = string.replace(".:Get::EntryContent:.",".:GetEntryContent:.")
......@@ -56,6 +58,18 @@ def ReplacePatterns(string):
string = string.replace(".:For::EntryTags:",".:ForEntryTags:")
string = string.replace(".:For::BlogDates:",".:ForBlogDates:")
string = string.replace(".:PagesList:",".:ForPages:")
string = string.replace("{0[tag]}", "{value}")
string = string.replace("{0[categoryPath]}", "{path}")
string = string.replace("{0[relativeOrigin]}", '')
string = string.replace("{0[date]}", "{value}")
string = string.replace("{0[item]}", "{value}")
string = string.replace("{0[pageUrl]}", "{path}")
string = string.replace("{0[dateUrl]}", "{path}")
string = string.replace("{0[count]}", "{count}")
string = string.replace("{0[weight]}", "{weight}")
string = string.replace("{0[destinationPageUrl]}", "{path}")
string = string.replace("{0[pageNumber]}", "{page_number}")
return string
......@@ -72,47 +86,63 @@ def to_camel_case(name):
return output
''' Processing blog configuration '''
try:
blog_configuration = yaml.load(open(os.getcwd()+"/blog_configuration.yaml",'r').read())
if not "blogConfiguration.yaml" in os.listdir():
try:
blog_configuration = yaml.load(open(os.getcwd()+"/blog_configuration.yaml",'r').read())
except FileNotFoundError:
die(messages.no_blog_configuration)
except FileNotFoundError:
die(messages.no_blog_configuration)
except PermissionError:
die(messages.no_blog_configuration)
except PermissionError:
die(messages.no_blog_configuration)
except yaml.scanner.ScannerError:
except yaml.scanner.ScannerError:
die(messages.possible_malformed_blog_configuration)
blog_configuration = dict()
for rootkey in blog_configuration:
if rootkey != "path":
blog_configuration[ to_camel_case(rootkey) ] = blog_configuration[rootkey]
else:
blog_configuration["path"] = dict()
for pathkey in blog_configuration["path"]:
if pathkey == "entry_file_name":
blog_configuration["path"][ to_camel_case(pathkey) ] = blog_configuration["path"][pathkey].replace("{entry_id}","{0[entryId]}")
elif pathkey == "index_file_name":
blog_configuration["path"][ to_camel_case(pathkey) ] = blog_configuration["path"][pathkey].replace("{page_number}","{0[pageNumber]}")
else:
blog_configuration["path"][ to_camel_case(pathkey) ] = blog_configuration["path"][pathkey]
blog_configuration.pop("threadOrder")
blog_configuration["reverseThreadOrder"] = True
for key in ["thread_order", "textEditor"]:
try:
blog_configuration.pop(key)
stream = codecs.open('blogConfiguration.yaml', 'w',encoding="utf-8")
yaml.dump(blog_configuration, stream, default_flow_style=False, allow_unicode=True)
except KeyError:
pass
new_keys = {
"text_editor":"nano",
"reverse_thread_order" : True,
"markup_language": "Markdown",
"disable_threads": "",
"disable_main_thread": False,
"disable_archives": False,
"disable_categories": False,
"disable_single_entries": False,
"path_encoding":"utf-8",
"code_highlight_css_override":False
}
new_path_keys = {
"entries_sub_folders":"",
"categories_sub_folders":"",
"dates_sub_folders":""
}
for key in new_keys.keys():
if not key in blog_configuration.keys():
blog_configuration[key] = new_keys[key]
if not "path" in blog_configuration.keys():
blog_configuration["path"] = {}
for key in new_path_keys.keys():
if not key in blog_configuration["path"].keys():
blog_configuration["path"][key] = new_path_keys[key]
stream = codecs.open('blog_configuration.yaml', 'w',encoding="utf-8")
yaml.dump(blog_configuration, stream, default_flow_style=False, allow_unicode=True)
''' Processing entries '''
for entry_filename in yield_entries_content():
raw_data = open(os.getcwd()+"/entries/"+entryFilename,'r').read()
raw_data = open(os.getcwd()+"/entries/"+entry_filename,'r').read()
try:
metadata = yaml.load(raw_data.split("---\n")[0])
metadata["title"] = metadata.pop("entry_name")
......@@ -126,10 +156,10 @@ for entry_filename in yield_entries_content():
else:
pass
output = yaml.dump(metadata, default_flow_style=False, allow_unicode=True) + "---\n"
output = yaml.dump(metadata, default_flow_style=False, allow_unicode=True) + "\n---VENC-BEGIN-PREVIEW---\n---VENC-END-PREVIEW---\n"
try:
output += replace_patterns(rawData.split("---\n")[1])
output += replace_patterns(raw_data.split("---\n")[1])
except IndexError:
notify(messages.empty_entry.format(entry_filename), "YELLOW")
......@@ -145,11 +175,7 @@ for chunk in os.listdir("theme/chunks"):
update = replace_patterns(raw_data)
stream.write(update)
for chunk in ["audio.html","video.html"]:
shutil.copy(os.path.expanduser("~/.local/share/VenC/themes/dummy/chunks/"+chunk), "theme/chunks/"+chunk)
''' Cleaning '''
try:
os.remove('blog_configuration.yaml')
except:
pass
print("Done.")
......@@ -27,6 +27,7 @@ from venc2.datastore.entry import yield_entries_content
from venc2.datastore.entry import Entry
from venc2.datastore.metadata import build_categories_tree
from venc2.datastore.metadata import MetadataNode
from venc2.helpers import notify
from venc2.patterns.non_contextual import embed_content
def merge(iterable, argv):
......@@ -47,7 +48,9 @@ class DataStore:
self.max_category_weight = 1
self.categories_leaves = []
self.embed_providers = {}
self.html_categories_tree = None
self.html_categories_tree = {}
self.html_categories_leaves = {}
self.html_blog_dates = {}
''' Entry index is different from entry id '''
entry_index = 0
......@@ -71,7 +74,12 @@ class DataStore:
''' Update entriesPerCategories '''
sub_folders = urllib.parse.quote(self.blog_configuration["path"]["categories_sub_folders"]+'/', encoding=self.blog_configuration["path_encoding"])
try:
sub_folders = urllib.parse.quote(self.blog_configuration["path"]["categories_sub_folders"]+'/', encoding=self.blog_configuration["path_encoding"])
except UnicodeEncodeError as e:
sub_folders = self.blog_configuration["path"]["categories_sub_folders"]+'/'
notify("\"{0}\": ".format(sub_folders)+str(e), color="YELLOW")
sub_folders = sub_folders if sub_folders != '/' else ''
build_categories_tree(entry_index, self.entries[-1].raw_categories, self.entries_per_categories, self.categories_leaves, self.max_category_weight, self.set_max_category_weight, encoding=self.blog_configuration["path_encoding"], sub_folders=sub_folders)
entry_index += 1
......@@ -79,12 +87,17 @@ class DataStore:
''' Setup BlogDates Data '''
self.blog_dates = list()
for node in self.entries_per_dates:
sub_folders = urllib.parse.quote(self.blog_configuration["path"]["dates_sub_folders"]+'/', encoding=self.blog_configuration["path_encoding"])
try:
sub_folders = urllib.parse.quote(self.blog_configuration["path"]["dates_sub_folders"]+'/', encoding=self.blog_configuration["path_encoding"])
except UnicodeEncodeError as e:
sub_folders = self.blog_configuration["path"]["dates_sub_folders"]+'/'
notify("\"{0}\": ".format(sub_folders)+str(e), color="YELLOW")
sub_folders = sub_folders if sub_folders != '/' else ''
self.blog_dates.append({
"date":node.value,
"dateUrl": ".:GetRelativeOrigin:."+sub_folders+node.value,
"value":node.value,
"path": ".:GetRelativeOrigin:."+sub_folders+node.value,
"count": node.count,
"weight": node.weight
})
......@@ -155,7 +168,6 @@ class DataStore:
def get_entry_year(self, argv=list()):
return self.entries[self.requested_entry_index].date.year
def get_entry_month(self, argv=list()):
return self.entries[self.requested_entry_index].date.month
......@@ -205,8 +217,12 @@ class DataStore:
return self.blog_configuration["author_email"]
def for_blog_dates(self, argv):
dates = [o for o in self.blog_dates if o["date"] not in self.disable_threads]
return merge(dates, argv)
key = ''.join(argv)
if not key in self.html_blog_dates.keys():
dates = [o for o in self.blog_dates if o["value"] not in self.disable_threads]
self.html_blog_dates[key] = merge(dates, argv)
return self.html_blog_dates[key]
def get_root_page(self, argv):
return ".:GetRelativeOrigin:."+self.blog_configuration["path"]["index_file_name"].format(**{"page_number":''})
......@@ -218,7 +234,7 @@ class DataStore:
continue
variables = {
"item" : node.value,
"value" : node.value,
"count" : node.count,
"weight" : round(node.weight / self.max_category_weight,2),
"path" : node.path
......@@ -242,9 +258,9 @@ class DataStore:
return output_string + closing_node
def tree_for_entry_categories(self, argv):
# compute once categories tree and deliver baked html
if self.entries[self.requested_entry_index].html_categories_tree == None:
self.entries[self.requested_entry_index].html_categories_tree = self.build_html_categories_tree(
key = ''.join(argv)
if not key in self.entries[self.requested_entry_index].html_categories_tree.keys():
self.entries[self.requested_entry_index].html_categories_tree[key] = self.build_html_categories_tree(
argv[0], #opening_node
argv[1], #opening_branch
argv[2], #closing_branch
......@@ -252,12 +268,13 @@ class DataStore:
self.entries[self.requested_entry_index].categories_tree
)
return self.entries[self.requested_entry_index].html_categories_tree
return self.entries[self.requested_entry_index].html_categories_tree[key]
def tree_for_blog_categories(self, argv):
key = ''.join(argv)
# compute once categories tree and deliver baked html
if self.html_categories_tree == None:
self.html_categories_tree = self.build_html_categories_tree(
if not key in self.html_categories_tree.keys():
self.html_categories_tree[key] = self.build_html_categories_tree(
argv[0], #opening_node
argv[1], #opening_branch
argv[2], #closing_branch
......@@ -265,29 +282,45 @@ class DataStore:
self.entries_per_categories
)
return self.html_categories_tree
return self.html_categories_tree[key]
def for_entry_tags(self, argv):
return merge(self.entries[self.requested_entry_index].tags, argv)
key = ''.join(argv)
if not key in self.entries[self.requested_entry_index].html_tags.keys():
self.entries[self.requested_entry_index].html_tags[key] = merge(self.entries[self.requested_entry_index].tags, argv)
return self.entries[self.requested_entry_index].html_tags[key]
def for_entry_authors(self, argv):
return merge(self.entries[self.requested_entry_index].authors, argv)
key = ''.join(argv)
if not key in self.entries[self.requested_entry_index].html_authors.keys():
self.entries[self.requested_entry_index].html_authors[key] = merge(self.entries[self.requested_entry_index].authors, argv)
return self.entries[self.requested_entry_index].html_authors[key]
""" TODO in 2.x.x: Access {count} and {weight} from LeavesForEntrycategories by taking benefit of preprocessing. """
def leaves_for_entry_categories(self, argv):
return merge(self.entries[self.requested_entry_index].categories_leaves, argv)
key = ''.join(argv)
if not key in self.entries[self.requested_entry_index].html_categories_leaves.keys():
self.entries[self.requested_entry_index].html_categories_leaves[key] = merge(self.entries[self.requested_entry_index].categories_leaves, argv)
return self.entries[self.requested_entry_index].html_categories_leaves[key]
def leaves_for_blog_categories(self, argv):
items = []
for node in self.categories_leaves:
items.append({
"item" : node.value,
"count" : node.count,
"weight" : round(node.weight / self.max_category_weight,2),
"path" : node.path
})
key = ''.join(argv)
if not key in self.html_categories_leaves.keys():
items = []
for node in self.categories_leaves:
items.append({
"value" : node.value,
"count" : node.count,
"weight" : round(node.weight / self.max_category_weight,2),
"path" : node.path
})
return merge(items, argv)
self.html_categories_leaves[key] = merge(items, argv)
return self.html_categories_leaves[key]
def cache_embed_exists(self, link):
cache_filename = hashlib.md5(link.encode('utf-8')).hexdigest()
......
......@@ -51,7 +51,8 @@ def get_blog_configuration():
"disable_archives",
"disable_categories",
"disable_single_entries",
"path_encoding"
"path_encoding",
"code_highlight_css_override"
]
everything_is_okay = True
......@@ -77,9 +78,12 @@ def get_blog_configuration():
everything_is_okay = False
notify(messages.missing_mandatory_field_in_blog_conf.format(field),"RED")
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
try:
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blog_configuration.yaml"),"RED")
except KeyError:
pass
if not everything_is_okay:
exit()
......
......@@ -112,7 +112,7 @@ class Entry:
die(messages.missing_mandatory_field_in_entry.format("authors", self.id))
try:
self.tags = [ {"tag":e} for e in list(metadata["tags"].split(",") if metadata["tags"] != str() else list())]
self.tags = [ {"value":e} for e in list(metadata["tags"].split(",") if metadata["tags"] != str() else list())]
except KeyError:
die(messages.missing_mandatory_field_in_entry.format("tags", self.id))
......@@ -123,10 +123,12 @@ class Entry:
}
sf = paths["entries_sub_folders"].format(**params)
sf = sf+'/' if sf != '' else ''
self.url = ".:GetRelativeOrigin:."+urllib.parse.quote(
sf+paths["entry_file_name"].format(**params),
encoding=encoding
)
try:
self.url = ".:GetRelativeOrigin:."+urllib.parse.quote(sf+paths["entry_file_name"].format(**params), encoding=encoding)
except UnicodeEncodeError as e:
self.url = ".:GetRelativeOrigin:."+sf+paths["entry_file_name"].format(**params)
notify("\"{0}\": ".format(sf+paths["entry_file_name"].format(**params))+str(e), color="YELLOW")
self.categories_leaves = list()
self.raw_categories = [ c.strip() for c in metadata["categories"].split(',')]
try:
......@@ -147,14 +149,24 @@ class Entry:
self.categories_tree = []
build_categories_tree(-1, self.raw_categories, self.categories_tree, None, -1, encoding=encoding, sub_folders=paths["categories_sub_folders"])
self.html_categories_tree = None
self.html_categories_tree = {}
self.html_tags = {}
self.html_authors = {}
self.html_categories_leaves = {}
def sort_entry(entry_id):
try:
return int(entry_id)
except ValueError:
return False
''' Iterate through entries folder '''
def yield_entries_content():
try:
for filename in sorted(
os.listdir(os.getcwd()+"/entries"),
key = lambda entry_id : int(entry_id.split("__")[0])
key = lambda entry_id : sort_entry(entry_id.split("__")[0])
):
exploded_filename = filename.split("__")
try:
......
......@@ -20,6 +20,8 @@
import datetime
import urllib.parse
from venc2.helpers import notify
class MetadataNode:
def __init__(self, value, entry_index):
self.count = 1
......@@ -47,8 +49,12 @@ def build_categories_tree(entry_index, input_list, output_tree, output_leaves, m
root.append(MetadataNode(node_name, entry_index))
if output_leaves != None and node_name == leave:
output_leaves.append(root[-1])
root[-1].path = ".:GetRelativeOrigin:."+urllib.parse.quote(path, encoding=encoding)
try:
root[-1].path = ".:GetRelativeOrigin:."+urllib.parse.quote(path, encoding=encoding)
except UnicodeEncodeError as e:
root[-1].path = ".:GetRelativeOrigin:."+path
notify("\"{0}\": ".format(root[-1].path)+str(e), color="YELLOW")
root = root[-1].childs
else:
......
......@@ -139,7 +139,7 @@ class Thread:
output += string.format(
**{
"page_number":str(page_number),
"page_url": self.filename.format(**{"page_number": (str() if page_number == 0 else page_number) })
"path": self.filename.format(**{"page_number": (str() if page_number == 0 else page_number) })
}
) + separator
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment