Commit e404d748 authored by Denis Salem's avatar Denis Salem
Browse files

add test

parent 86d937c9
......@@ -37,3 +37,4 @@ disable_single_entries: true
banner: 'https://download.tuxfamily.org/dsalem/img/2017_-_Denis_Salem_-_CC_By_SA_-_VenC-logo.svg'
path_encoding: 'windows-1252'
server_port: 8888
sort_by: 'title'
......@@ -97,7 +97,6 @@
DONE | More verbosity about wrong pattern arguments.
DONE | More verbosity about malformed entry.
DONE | Clear white spaces in error messages.
DONE | Speed up blog exportation (almost four times faster!).
DONE | Implement RelativeLocation.
DONE | Optimisation of categories tree.
DONE | Code should be a little bit more PEP 8 compliant.
......@@ -143,8 +142,11 @@
DONE | Split unit-test from main branch
DONE | Support ForPage in entries thread.
DONE | Escape patterns
TODO | Sort by metadata.
TODO | Update command help with.
DONE | Update command help with.
DONE | Handle ftp encoding error.
DONE | Sort by metadata.
WIP | Speed up blog exportation (almost four times faster!).
WIP | Chapters engine.
TODO | Open with navigator manual.
TODO | Client-side search engine.
TODO | Fix unit test.
......@@ -157,7 +159,8 @@
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
# version 2.x.x
OPT | Speed up pattern processing.
TODO | Make some dependencies optional.
TODO | Split themes from main branch
TODO | Add translation for Deutsch and Spanish.
TODO | Add docutils reStructuredText parser settings override.
......@@ -168,3 +171,4 @@
TODO | Add nice 404 errors page.
TODO | Improve Feed.
TODO | Print out what's going on while FTP transfert.
TODO | Fix gvim / -ex
......@@ -24,6 +24,11 @@ from venc2.helpers import notify
from venc2.l10n import messages;
from venc2.l10n import locale_err;
import cProfile, pstats, io
pr = cProfile.Profile()
pr.enable()
# ... do something ...
if locale_err:
notify(messages.cannot_get_current_locale, color="YELLOW")
......@@ -81,3 +86,10 @@ if sys.argv[1:] != []:
else:
notify(messages.nothing_to_do,"YELLOW")
pr.disable()
s = io.StringIO()
sortby = 'cumulative'
ps = pstats.Stats(pr, stream=s).sort_stats(sortby)
ps.print_stats()
print(s.getvalue())
......@@ -84,7 +84,6 @@ non_contextual_pattern_names_blog = {
"GetBlogURL" : "get_blog_url",
"GetBlogLanguage" : "get_blog_language",
"GetAuthorEmail" : "get_author_email",
"GetRootPage" : "get_root_page",
# Extra metadata getter
"GetBlogMetadataIfExists" : "get_blog_metadata_if_exists",
......@@ -118,6 +117,7 @@ contextual_pattern_names = {
"GetPreviousPage" : "get_previous_page",
"ForPages" : "for_pages",
"GetStyleSheets" : code_highlight.get_style_sheets,
"GetRootPage" : datastore.get_root_page,
**extra_contextual_pattern_names
}
......@@ -159,6 +159,7 @@ def export_blog(argv=list()):
# Blacklist contextual patterns
for pattern_name in contextual_pattern_names.keys():
processor.blacklist.append(pattern_name)
processor.blacklist.append("Escape")
notify(messages.pre_process)
......
......@@ -60,7 +60,7 @@ def new_entry(argv):
entry = dict()
raw_entry_date = datetime.datetime.now()
try:
entry["ID"] = max([ int(filename.split("__")[0]) for filename in yield_entries_content()]) + 1
entry["ID"] = max([ int(filename.split("__")[0]) for filename in yield_entries_content('id')]) + 1
except ValueError:
entry["ID"] = 1
......@@ -140,7 +140,8 @@ def new_blog(argv):
"reverse_thread_order": True,
"markup_language": "Markdown",
"path_encoding": "utf-8",
"server_port": 8888}
"server_port": 8888,
"sort_by":"id"}
for folder_name in argv:
try:
os.mkdir(folder_name)
......
......@@ -30,6 +30,7 @@ def print_help(argv=None):
print("-ne\t--new-entry <\""+messages.arg_entry_name+"\"> [\""+messages.arg_template_name+"\"]")
print("-xb\t--export-blog ["+messages.theme_name+"]")
print("-ex\t--edit-and-xport <\""+messages.arg_input_filename+"\">")
print("-s\t--serv")
print("-xftp\t--export-via-ftp")
print("-rc\t--remote-copy")
print("-h\t--help")
......
......@@ -31,6 +31,7 @@ def remote_copy(argv=list()):
blog_configuration = get_blog_configuration()
try:
ftp = ftplib.FTP(blog_configuration["ftp_host"])
ftp.encoding='latin-1'
except socket.gaierror as e:
die(str(e))
......@@ -46,30 +47,30 @@ def remote_copy(argv=list()):
notify(messages.copy_to_ftp_directory)
ftp_export_recursively(os.getcwd()+"/blog", ftp)
except ftplib.error_perm as e:
die(str(e))
except TimeoutError as e:
die(str(e))
except ftplib.error_perm as e:
die(str(e), color="YELLOW")
def ftp_export_recursively(origin, ftp):
folder = os.listdir(origin)
for item in folder:
if os.path.isdir(origin+"/"+item):
try:
ftp.mkd(item)
try:
ftp.mkd(item)
except ftplib.error_perm as e:
if not ": File exists" in str(e.args):
raise
ftp.cwd(ftp.pwd()+"/"+item)
ftp_export_recursively(origin+"/"+item, ftp)
ftp.cwd(ftp.pwd()[:-len("/"+item)])
except:
try:
ftp.cwd(ftp.pwd()+"/"+item)
ftp_export_recursively(origin+"/"+item, ftp)
ftp.cwd(ftp.pwd()[:-len("/"+item)])
except:
raise
except Exception as e:
notify(item+": "+str(e), color="YELLOW")
else:
ftp.storbinary("STOR "+ftp.pwd()+"/"+item, open(origin+"/"+item, 'rb'))
......@@ -77,6 +78,7 @@ def ftp_export_recursively(origin, ftp):
def ftp_clean_destination(ftp):
listing = list()
listing = ftp.nlst()
for item in listing:
if item not in ['.','..']:
try:
......
......@@ -28,6 +28,7 @@ from venc2.datastore.entry import yield_entries_content
from venc2.datastore.entry import Entry
from venc2.datastore.metadata import build_categories_tree
from venc2.datastore.metadata import MetadataNode
from venc2.datastore.metadata import Chapter
from venc2.helpers import notify
from venc2.patterns.non_contextual import get_embed_content
......@@ -53,40 +54,47 @@ class DataStore:
self.html_categories_leaves = {}
self.html_blog_dates = {}
self.generation_timestamp = datetime.datetime.now()
self.chapters = {}
''' Entry index is different from entry id '''
entry_index = 0
# Build entries
for filename in yield_entries_content():
if len(self.entries):
self.entries.append(Entry(filename, self.blog_configuration["path"], previous_entry = self.entries[-1], encoding=self.blog_configuration["path_encoding"]))
self.entries[-2].next_entry = self.entries[-1]
self.entries.append(Entry(filename, self.blog_configuration["path"], encoding=self.blog_configuration["path_encoding"]))
self.entries = sorted(self.entries, key = lambda entry : self.sort_by(entry))
else:
self.entries.append(Entry(filename, self.blog_configuration["path"], encoding=self.blog_configuration["path_encoding"]))
''' Update entriesPerDates '''
for entry_index in range(0, len(self.entries)):
if entry_index > 0:
self.entries[entry_index-1].next_entry = self.entries[entry_index]
self.entries[entry_index].previous_entry = self.entries[entry_index-1]
# Update entriesPerDates
if self.blog_configuration["path"]["dates_directory_name"] != '':
formatted_date = self.entries[-1].date.strftime(self.blog_configuration["path"]["dates_directory_name"])
entries_index = self.get_entries_index_for_given_date(formatted_date)
if entries_index != None:
self.entries_per_dates[entries_index].count +=1
self.entries_per_dates[entries_index].related_to.append(entry_index)
formatted_date = self.entries[entry_index].date.strftime(self.blog_configuration["path"]["dates_directory_name"])
entries_indexes = self.get_entries_index_for_given_date(formatted_date)
if entries_indexes != None:
self.entries_per_dates[entries_indexes].count +=1
self.entries_per_dates[entries_indexes].related_to.append(entry_index)
else:
self.entries_per_dates.append(MetadataNode(formatted_date, entry_index))
try:
self.update_chapters(self.entries[entry_index].chapter, entry_index)
except AttributeError:
pass
''' Update entriesPerCategories '''
# Update entriesPerCategories
try:
sub_folders = urllib.parse.quote(self.blog_configuration["path"]["categories_sub_folders"]+'/', encoding=self.blog_configuration["path_encoding"])
except UnicodeEncodeError as e:
sub_folders = self.blog_configuration["path"]["categories_sub_folders"]+'/'
notify("\"{0}\": ".format(sub_folders)+str(e), color="YELLOW")
sub_folders = sub_folders if sub_folders != '/' else ''
build_categories_tree(entry_index, self.entries[-1].raw_categories, self.entries_per_categories, self.categories_leaves, self.max_category_weight, self.set_max_category_weight, encoding=self.blog_configuration["path_encoding"], sub_folders=sub_folders)
entry_index += 1
build_categories_tree(entry_index, self.entries[entry_index].raw_categories, self.entries_per_categories, self.categories_leaves, self.max_category_weight, self.set_max_category_weight, encoding=self.blog_configuration["path_encoding"], sub_folders=sub_folders)
''' Setup BlogDates Data '''
# Setup BlogDates Data
self.blog_dates = list()
for node in self.entries_per_dates:
try:
......@@ -104,6 +112,17 @@ class DataStore:
"weight": node.weight
})
def update_chapters(self, chapter, entry_index):
pass
def sort_by(self, entry):
try:
return getattr(entry, self.blog_configuration["sort_by"])
except AttributeError:
print("FAILED")
return ''
def set_max_category_weight(self, value):
self.max_category_weight = value
return value
......
......@@ -55,7 +55,8 @@ def get_blog_configuration():
"code_highlight_css_override",
"server_port",
"disable_rss_feed",
"disable_atom_feed"
"disable_atom_feed",
"sort_by"
]
everything_is_okay = True
......@@ -86,6 +87,10 @@ def get_blog_configuration():
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blog_configuration.yaml"),"RED")
if blog_configuration["sort_by"] in ['', None]:
blog_configuration["sort_by"] = "id"
except KeyError:
pass
......
......@@ -56,8 +56,8 @@ class EntryWrapper:
die(messages.missing_entry_content_inclusion)
class Entry:
def __init__(self, filename, paths, previous_entry = None, encoding="utf-8"):
self.previous_entry = previous_entry
def __init__(self, filename, paths, encoding="utf-8"):
self.previous_entry = None
self.next_entry = None
# Loading
......@@ -99,7 +99,7 @@ class Entry:
metadata[key] = ''
self.filename = filename
self.id = filename.split('__')[0]
self.id = int(filename.split('__')[0])
raw_date = filename.split('__')[1].split('-')
self.date = datetime.datetime(
......@@ -170,20 +170,10 @@ class Entry:
self.html_authors = {}
self.html_categories_leaves = {}
def sort_entry(entry_id):
try:
return int(entry_id)
except ValueError:
return False
''' Iterate through entries folder '''
def yield_entries_content():
try:
for filename in sorted(
os.listdir(os.getcwd()+"/entries"),
key = lambda entry_id : sort_entry(entry_id.split("__")[0])
):
for filename in os.listdir(os.getcwd()+"/entries"):
exploded_filename = filename.split("__")
try:
date = exploded_filename[1].split('-')
......
......@@ -22,6 +22,14 @@ import urllib.parse
from venc2.helpers import notify
class Chapter:
def __init__(self, chapter, entry):
self.sub_chapters = []
self.chapter = chapter
self.url = entry.url
self.title = entry.title
class MetadataNode:
def __init__(self, value, entry_index):
self.count = 1
......
......@@ -83,19 +83,19 @@ class EntriesThread(Thread):
next_entry = self.current_entry.next_entry
previous_entry = self.current_entry.previous_entry
for i in range(0, list_lenght):
if entry_next != None:
if next_entry != None:
params["entry_id"] = next_entry.id
params["entry_title"] = next_entry.title
params["path"] = next_entry.url
output += string.format(**params) + separator
next_entry = entry_next.next_entry
next_entry = next_entry.next_entry
if entry_previous != None:
params["entry_id"] = entry_previous.id
params["entry_title"] = entry_previous.title
params["path"] = entry_previous.url
if previous_entry != None:
params["entry_id"] = previous_entry.id
params["entry_title"] = previous_entry.title
params["path"] = previous_entry.url
output = string.format(**params) + separator + output
entry_previous = entry_previous.previous_entry
previous_entry = previous_entry.previous_entry
return output[:-len(separator)]
......
#! /usr/bin/python
import time
iteration = 1
string = ".:foo::bar:. .:IfInThread:: .:boo:: .:foo::bar:. :. :: lolilol :. .:END:. .:TEST AGAIN:: .:WITHIN:. :."
print('>', string)
def algo1():
op = []
cp = []
i = 0
l = len(string)
output = string
offset=0
while i < l:
i = output.find(".:", i)
if i == -1:
i=0
break
op.append(i)
i+=2
while i < l:
i = output.find(":.", i)
if i == -1:
i=0
break
cp.append(i)
i+=2
l = len(op)
while l:
for i in range(0, l):
print(output, i, l)
try:
cmpr = cp[i] < op[i+1]
except Exception as e:
if len(cp) != l:
raise e
if cmpr:
vop, vcp = op[i], cp[i]
fields = [field.strip() for field in output[vop+2:vcp].split("::") if field != '']
new_string = "["+'---'.join(fields)+"]"
output = output[0:vop] + new_string + output[vcp+2:]
offset = len(new_string) - (vcp + 2 - vop)
for j in range(i+1,l):
op[j]+=offset
cp[j]+=offset
op.pop(i)
cp.pop(i)
l = len(op)
break
return output
def algo2(string):
op = []
cp = []
l = len(string)
fields=[]
for i in range(0, l):
if string[i:i+2] == ".:":
op.append(i)
elif string[i:i+2] == ":.":
cp.append(i)
if len(cp) <= len(op) and len(cp) != 0 and len(op) != 0:
if op[-1] < cp[-1]:
fields = [field.strip() for field in string[op[-1]+2:cp[-1]].split("::") if field != '']
return algo2(string[:op[-1]]+"["+'---'.join(fields)+"]"+string[cp[-1]+2:])
return string
t = time.time()
for i in range(0,iteration):
c = algo1()
print(c)
print("ALGO1", time.time()-t)
t = time.time()
for j in range(0,iteration):
s = algo2(string)
print(s)
print("ALGO2", time.time()-t)
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment