Commit 74467f85 authored by dsalem's avatar dsalem
Browse files

Fix chapters with parallel processing

parent bc2d10a4
......@@ -67,14 +67,13 @@
DONE | Split process_markup_langage from pattern processor.
DONE | Fix locale import exception. (#12)
DONE | Explicit needles replacement.
DONE | Add CPU threads support.
WIP | Enable / Disable entries title in ForPage for gentle and tessellation
WIP | Generate entry inner title index.
WIP | Add If*MetadataIsTrue.
TODO | Add nice 404 errors page.
TODO | Update doc accordingly.
TODO | Add ForDictMetadata
DONE | Add CPU threads support.
# version 2.0.2
DONE | Fix for_entry_metadata; catching unavailable contextual pattern.
......
......@@ -135,6 +135,8 @@ def setup_pattern_processor(pattern_map):
def worker_process_non_contextual_entry_patterns(payload):
datastore = payload["datastore"]
datastore.in_child_process = True
worker_id = payload["worker_id"]
workers_count = datastore.workers_count
......@@ -158,11 +160,8 @@ def worker_process_non_contextual_entry_patterns(payload):
pattern_processor.process(entry.preview)
process_markup_language(entry.preview, markup_language)
# ~ print(entry.title)
pattern_processor.process(entry.content)
process_markup_language(entry.content, markup_language, entry)
# ~ if entry.title == "Notion de limite avec Zénon et Euclide":
# ~ print(entry.content.string)
entry.html_wrapper = deepcopy(theme.entry)
pattern_processor.process(entry.html_wrapper.processed_string)
......@@ -184,14 +183,14 @@ def process_non_contextual_patterns(init_theme_argv):
patterns_map = PatternsMap(datastore, code_highlight, theme)
pattern_processor = setup_pattern_processor(patterns_map)
chunks_len = (len(datastore.entries)//datastore.workers_count)+1
datastore.chunks_len = (len(datastore.entries)//datastore.workers_count)+1
workers = []
context_chunks = []
if datastore.workers_count > 1:
for i in range(0, datastore.workers_count):
context_chunks.append({
"datastore" : split_datastore(datastore, chunks_len),
"datastore" : split_datastore(datastore),
"init_theme_argv": init_theme_argv,
"worker_id": i
})
......@@ -207,6 +206,7 @@ def process_non_contextual_patterns(init_theme_argv):
datastore.entries = []
for chunk in entries:
datastore.entries += chunk[0]
#todo merge codehighlight
else:
worker_process_non_contextual_entry_patterns({
......@@ -244,7 +244,7 @@ def export_blog(argv=list()):
notify("├─ "+messages.pre_process)
theme, theme_folder, code_highlight, patterns_map = process_non_contextual_patterns(argv)
# cleaning directory
shutil.rmtree("blog", ignore_errors=False, onerror=rm_tree_error_handler)
os.makedirs("blog")
......
......@@ -30,7 +30,7 @@ from urllib.parse import quote as urllib_parse_quote
from venc2.datastore.configuration import get_blog_configuration
from venc2.datastore.entry import yield_entries_content
from venc2.datastore.entry import AdjacentEntryMetadata
from venc2.datastore.entry import MinimalEntryMetadata
from venc2.datastore.entry import Entry
from venc2.datastore.metadata import build_categories_tree
from venc2.datastore.metadata import MetadataNode
......@@ -56,16 +56,17 @@ def merge(iterable, argv):
raise e
def split_datastore(datastore, chunk_len):
def split_datastore(datastore):
entries = datastore.entries
datastore.entries = []
chunk = deepcopy(datastore)
chunk.entries = entries[:chunk_len]
datastore.entries = entries[chunk_len:]
chunk.entries = entries[:datastore.chunks_len]
datastore.entries = entries[datastore.chunks_len:]
return chunk
class DataStore:
def __init__(self):
self.in_child_process = False
notify("┌─ "+messages.loading_data)
self.root_page = None
self.blog_configuration = get_blog_configuration()
......@@ -120,14 +121,17 @@ class DataStore:
# Build entries
try:
jsonld_callback = self.entry_to_jsonld_callback if (self.enable_jsonld or self.enable_jsonp) else None
index = 0
for filename in yield_entries_content():
self.entries.append(Entry(
index,
filename,
self.blog_configuration["path"],
jsonld_callback,
self.blog_configuration["path"]["archives_directory_name"],
self.path_encoding
))
index+=1
# Might happen during Entry creation.
except MalformedPatterns as e:
......@@ -142,8 +146,8 @@ class DataStore:
for entry_index in range(0, len(self.entries)):
current_entry = self.entries[entry_index]
if entry_index > 0:
self.entries[entry_index-1].next_entry = AdjacentEntryMetadata(current_entry)
current_entry.previous_entry = AdjacentEntryMetadata(self.entries[entry_index-1])
self.entries[entry_index-1].next_entry = MinimalEntryMetadata(current_entry)
current_entry.previous_entry = MinimalEntryMetadata(self.entries[entry_index-1])
# Update entriesPerDates
if path_archives_directory_name != '':
......@@ -167,12 +171,15 @@ class DataStore:
notify("\"{0}\": ".format(path_categories_sub_folders)+str(e), color="YELLOW")
sub_folders = sub_folders if sub_folders != '/' else ''
# TODO : should not be done unless it's necessary
build_categories_tree(entry_index, current_entry.raw_categories, self.entries_per_categories, self.categories_leaves, self.max_category_weight, self.set_max_category_weight, encoding=self.path_encoding, sub_folders=sub_folders)
self.update_chapters(current_entry)
# build chapters index
path_chapters_sub_folders = self.blog_configuration["path"]["chapters_sub_folders"]
path_chapter_folder_name = self.blog_configuration["path"]["chapter_directory_name"]
#TODO: Might be not safe, must test level if is actually an int. Test as well the whole sequence.
for chapter in sorted(self.raw_chapters.keys(), key = lambda x : int(x.replace('.', ''))):
......@@ -196,9 +203,10 @@ class DataStore:
except StopIteration:
if index in self.raw_chapters.keys():
# TODO: Replace this shitty bloc by a function call building path
entry = self.raw_chapters[index]
try:
path = "\x1a"+((path_chapters_sub_folders+'/' if path_chapters_sub_folders != '' else '')+path_chapter_folder_name).format(**{
"chapter_name" : self.raw_chapters[index].title,
"chapter_name" : entry.title,
"chapter_index" : index
})
try:
......@@ -218,7 +226,7 @@ class DataStore:
top.append(
Chapter(index, self.raw_chapters[index], path)
)
self.raw_chapters[index].chapter = top[-1]
entry.chapter = top[-1]
else:
top.append(
......@@ -497,7 +505,8 @@ class DataStore:
return self.html_chapters[key]
def build_html_chapters(self, argv, top, level):
#TODO : Raise MissingArgs if... missing args.
def build_html_chapters(self, argv, top, level):
lo, io, ic, lc = argv
if top == []:
return ''
......@@ -505,7 +514,7 @@ class DataStore:
path_encoding = self.path_encoding
output = lo.format(**{"level" :level})
for sub_chapter in top:
for sub_chapter in top:
output += io.format(**{
"index": sub_chapter.index,
"title": sub_chapter.entry.title,
......@@ -539,7 +548,7 @@ class DataStore:
chapter
))
else:
self.raw_chapters[chapter] = entry
self.raw_chapters[chapter] = MinimalEntryMetadata(entry)
def sort(self, entry):
try:
......@@ -699,10 +708,11 @@ class DataStore:
key = ''.join(argv[:2])
if not key in self.cache_get_chapter_attribute_by_index.keys():
try:
try:
self.cache_get_chapter_attribute_by_index[key] = getattr(self.raw_chapters[argv[1]].chapter, argv[0])
except KeyError as e:
raise PatternInvalidArgument(
"index",
argv[1],
......
......@@ -58,14 +58,17 @@ class EntryWrapper:
self.processed_string = ProcessedString(wrapper, filename, True)
class AdjacentEntryMetadata:
class MinimalEntryMetadata:
def __init__(self, entry):
self.id = entry.id
self.url = entry.url
self.title = entry.title
self.chapter = entry.chapter
self.index = entry.index
class Entry:
def __init__(self, filename, paths, jsonld_callback, date_format, encoding="utf-8", ):
def __init__(self, index, filename, paths, jsonld_callback, date_format, encoding="utf-8", ):
self.index = index
self.previous_entry = None
self.next_entry = None
self.chapter = None
......
......@@ -29,6 +29,7 @@ class Chapter:
self.sub_chapters = []
self.index = index
self.entry = entry
self.entry_index = entry.index
self.path = path
class MetadataNode:
......
#! /usr/bin/env python3
# Copyright 2016, 2020 Denis Salem
# Copyright 2016, 2021 Denis Salem
#
# This file is part of VenC.
#
......
#! /usr/bin/env python3
# Copyright 2016, 2020 Denis Salem
# Copyright 2016, 2021 Denis Salem
#
# This file is part of VenC.
#
......
......@@ -364,9 +364,7 @@ class Thread:
def do_iteration(self, entry):
global current_source
entry_wrapper = getattr(entry, self.content_type+"_wrapper")
entry_wrapper = getattr(entry, self.content_type+"_wrapper")
preprocessed_wrapper = getattr(entry, self.content_type+"_wrapper").processed_string
self.processor.process(preprocessed_wrapper, safe_process = True)
......
......@@ -27,7 +27,6 @@ class ChaptersThread(Thread):
super().__init__(prompt, datastore, theme, patterns_map)
self.filename = self.datastore.blog_configuration["path"]["index_file_name"]
self.entries_per_page = self.datastore.blog_configuration["entries_per_pages"]
self.folder_name = self.datastore.blog_configuration["path"]["chapter_directory_name"]
self.sub_folders = self.datastore.blog_configuration["path"]["chapters_sub_folders"]
self.relative_origin = str(''.join([ "../" for p in self.sub_folders.split('/') if p != ''])).replace("//",'/')
......@@ -48,12 +47,12 @@ class ChaptersThread(Thread):
else:
tree_special_char = '├'
notify(self.indentation_level+tree_special_char+"─ "+node.index+' '+node.entry.title+"...")
self.thread_name = node.entry.title
entry = self.datastore.entries[node.entry_index]
notify(self.indentation_level+tree_special_char+"─ "+node.index+' '+entry.title+"...")
self.thread_name = entry.title
self.export_path = "blog/"+self.sub_folders+'/'+self.folder_name
self.export_path = self.export_path.format(**{
"chapter_name" : node.entry.title,
"chapter_name" : entry.title,
"chapter_index": node.index
})
self.export_path = self.path_encode(self.export_path)
......@@ -71,7 +70,7 @@ class ChaptersThread(Thread):
output = []
output_append = output.append
for c in sub_chapters:
output_append(c.entry)
output_append(self.datastore.entries[c.entry_index])
if len(c.sub_chapters):
for sc_entries in self.extract_sub_chapters(c.sub_chapters):
output_append(sc_entries)
......@@ -85,7 +84,8 @@ class ChaptersThread(Thread):
for chapter_index in range(0, len(top)):
chapter = top[chapter_index]
self.organize_entries([ chapter.entry ] + self.extract_sub_chapters(chapter.sub_chapters))
entry = self.datastore.entries[chapter.entry_index]
self.organize_entries([ entry ] + self.extract_sub_chapters(chapter.sub_chapters))
self.setup_chapters_context(chapter_index, top, len(top))
super().do()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment