Commit 3ce22f3d authored by Denis Salem's avatar Denis Salem
Browse files

wip: improving prompt/refactorisation/import module optimisation/jsonld generation

parent 05c43a1f
......@@ -140,7 +140,7 @@
DONE | Add support for Atom feed.
DONE | Add blog generation timestamp.
DONE | Split unit-test from main branch
DONE | Support ForPage in entries thread.
DONE | Support ForPage in entries thread.
DONE | Update command help with.
DONE | Handle ftp encoding error.
DONE | Sort by metadata.
......@@ -155,6 +155,7 @@
DONE | fix FAQ categories separator is ' > ', not '>'
DONE | fix FAQ subfolders must not start with '/'
DONE | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
DONE | Refactor DatesThread to ArchivesThread.
TODO | Add definition for optional fields in blog configuration .
TODO | Open with navigator manual.
TODO | Client-side search engine.
......@@ -185,6 +186,7 @@
TODO | Add more informations into JSON-LD document.
TODO | Optimize JSON-LD processing.
TODO | Embed as much as possible microdata/JSON-LD into blog.
TODO | Automatically add entries categories to entries tags.
# version 3.x.x
TODO | Add support for plugin.
......
......@@ -35,11 +35,6 @@ from venc2.patterns.contextual import extra_contextual_pattern_names
from venc2.patterns.non_contextual import non_contextual_pattern_names
from venc2.patterns.processor import Processor
from venc2.patterns.processor import ProcessedString
from venc2.threads.categories import CategoriesThread
from venc2.threads.dates import DatesThread
from venc2.threads.entries import EntriesThread
from venc2.threads.feed import FeedThread
from venc2.threads.main import MainThread
from venc2.commands.remote import remote_copy
# Initialisation of environment
......@@ -212,18 +207,22 @@ def export_blog(argv=list()):
datastore.embed_providers = {}
# Starting second pass and exporting
from venc2.threads.main import MainThread
thread = MainThread(messages.export_main_thread, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_archives"]:
thread = DatesThread(messages.export_archives, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
from venc2.threads.archives import ArchivesThread
thread = ArchivesThread(messages.export_archives, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_categories"]:
from venc2.threads.categories import CategoriesThread
thread = CategoriesThread(messages.export_categories, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
if not datastore.blog_configuration["disable_single_entries"]:
from venc2.threads.entries import EntriesThread
thread = EntriesThread(messages.export_single_entries, datastore, theme, contextual_pattern_names, non_contextual_pattern_names_entry_keys)
thread.do()
......@@ -232,12 +231,6 @@ def export_blog(argv=list()):
code_highlight.export_style_sheets()
copy_recursively("extra/","blog/")
copy_recursively(theme_folder+"assets/","blog/")
if datastore.enable_jsonld:
import json
dump = json.dumps(datastore.blog_as_jsonld)
f = open("blog/blog.jsonld", 'w')
f.write(dump)
......
......@@ -57,8 +57,12 @@ class DataStore:
self.raw_chapters = {}
self.chapters_index = []
self.html_chapters = {}
self.entries_as_jsonld = {}
# Build JSON-LD doc if any
if self.enable_jsonld:
self.entries_as_jsonld = {}
self.root_site_to_jsonld()
# Build entries
try:
jsonld_callback = self.entry_to_jsonld_callback if self.enable_jsonld else None
......@@ -149,10 +153,6 @@ class DataStore:
"weight": node.weight
})
# Build JSON-LD doc if any
if self.enable_jsonld:
self.root_site_to_jsonld()
def root_site_to_jsonld(self):
if "https://schema.org" in self.blog_configuration.keys():
optionals = self.blog_configuration["https://schema.org"]
......@@ -160,9 +160,10 @@ class DataStore:
else:
optionals = {}
self.blog_as_jsonld = {
self.root_as_jsonld = {
"@context": "http://schema.org",
"@type": ["Blog","WebPage"],
"@id" : self.blog_configuration["blog_url"]+"/root.jsonld",
"name": self.blog_configuration["blog_name"],
"url": self.blog_configuration["blog_url"],
"description": self.blog_configuration["blog_description"],
......@@ -178,7 +179,6 @@ class DataStore:
"@type": "CreativeWork",
"name": self.blog_configuration["license"]
},
"breadcrumb" : [],
"blogPost" : [],
**optionals
}
......@@ -188,11 +188,11 @@ class DataStore:
optionals = entry.schemadotorg
else:
optionals = {}:
optionals = {}
doc = {
"@context": "http://schema.org",
"@type" : ["BlogPosting","WebPage"],
"@type" : ["BlogPosting", "WebPage"],
"@id" : self.blog_configuration["blog_url"]+"/entry"+str(entry.id)+".jsonld",
"keywords" : entry.raw_metadata["tags"],
"headline" : entry.title,
......@@ -201,10 +201,11 @@ class DataStore:
"inLanguage" : self.blog_configuration["blog_language"],
"author" : [{"name":author["value"], "@type": "Person"} for author in entry.authors],
"url" : entry.url.replace(".:GetRelativeOrigin:.", self.blog_configuration["blog_url"]+"/"),
"breadcrumb" : self.blog_as_jsonld["breadcrumb"],
"relatedLink" : [ {
"@type":"Url"
} for c in]
"breadcrumb" : {
"itemListElement": [
]
},
"relatedLink" : [ c["path"] for c in entry.categories_leaves],
**optionals
}
......
......@@ -90,7 +90,7 @@ class Entry:
setattr(self, key, metadata[key])
elif key == "https://schema.org" and metadata[key] != None:
setattr(self, "schemadotorg", metadata[key]
setattr(self, "schemadotorg", metadata[key])
else:
notify(messages.invalid_or_missing_metadata.format(key, filename), color="YELLOW")
......
......@@ -270,7 +270,16 @@ class Thread:
if self.columns_counter >= self.columns_number:
self.columns_counter = 0
# Must be called in child class
def iterate_through_pages(self):
for page in self.pages:
self.pre_iteration()
for entry in page:
self.setup_context(entry)
self.do_iteration(entry)
self.post_iteration()
# Must be called in child class
def do(self):
self.current_page = 0
self.page_number = 0
......@@ -289,10 +298,5 @@ class Thread:
stream.close()
else:
for page in self.pages:
self.pre_iteration()
for entry in page:
self.setup_context(entry)
self.do_iteration(entry)
self.iterate_through_pages()
self.post_iteration()
......@@ -22,10 +22,9 @@ import os
from venc2.prompt import notify
from venc2.threads import Thread
class DatesThread(Thread):
class ArchivesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
self.filename = self.datastore.blog_configuration["path"]["index_file_name"]
self.sub_folders = self.datastore.blog_configuration["path"]["dates_sub_folders"]
self.relative_origin = str("../"+''.join([ "../" for p in self.sub_folders.split('/') if p != ''])).replace("//",'/')
......@@ -50,6 +49,11 @@ class DatesThread(Thread):
])
super().do()
def JSONLD(self, argv):
if self.current_page == 0:
return '<script type="application/ld+json" src="archives.jsonld"></script>'
return ''
......
......@@ -22,7 +22,6 @@ import urllib.parse
from venc2.prompt import notify
from venc2.threads import Thread
from venc2.threads.feed import FeedThread
class CategoriesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
......@@ -32,10 +31,16 @@ class CategoriesThread(Thread):
self.export_path = "blog/"+self.datastore.blog_configuration["path"]["categories_sub_folders"]+'/'
self.relative_origin = ""
self.in_thread = True
if not self.datastore.blog_configuration["disable_rss_feed"]:
self.disable_rss_feed = self.datastore.blog_configuration["disable_rss_feed"]
self.disable_atom_feed = self.datastore.blog_configuration["disable_atom_feed"]
if (not self.disable_rss_feed) or (not self.disable_atom_feed):
from venc2.threads.feed import FeedThread
if not self.disable_rss_feed:
self.rss_feed = FeedThread(datastore, theme, patterns, forbidden, "rss")
if not self.datastore.blog_configuration["disable_atom_feed"]:
if not self.disable_atom_feed:
self.atom_feed = FeedThread(datastore, theme, patterns, forbidden, "atom")
def if_in_categories(self, argv):
......@@ -65,12 +70,11 @@ class CategoriesThread(Thread):
entries = [self.datastore.entries[entry_index] for entry_index in node.related_to]
self.organize_entries( entries[::-1] if self.datastore.blog_configuration["reverse_thread_order"] else entries )
super().do()
entries = sorted(entries, key = lambda entry : entry.id, reverse=True)[0:self.datastore.blog_configuration["feed_lenght"]]
if not self.datastore.blog_configuration["disable_rss_feed"]:
if not self.disable_rss_feed:
self.rss_feed.do(entries, self.export_path, self.relative_origin)
if not self.datastore.blog_configuration["disable_atom_feed"]:
if not self.disable_atom_feed:
self.atom_feed.do(entries, self.export_path, self.relative_origin)
self.do(root=node.childs)
......@@ -78,6 +82,12 @@ class CategoriesThread(Thread):
# Restore path
self.export_path = export_path
def JSONLD(self, argv):
if self.current_page == 0:
return '<script type="application/ld+json" src="categories.jsonld"></script>'
return ''
......
......@@ -120,7 +120,15 @@ class EntriesThread(Thread):
)
stream.write(output)
stream.close()
def iterate_through_pages(self):
for page in self.pages:
for entry in page:
self.pre_iteration()
self.setup_context(entry)
self.do_iteration(entry)
self.post_iteration()
def do(self):
self.page_number = 0
self.current_page = 0
......@@ -132,4 +140,9 @@ class EntriesThread(Thread):
self.do_iteration(entry)
self.post_iteration()
def JSONLD(self, argv):
if self.current_page == 0:
return '<script type="application/ld+json" src="entry'+str(self.current_entry.id)+'.jsonld"></script>'
return ''
......@@ -39,7 +39,7 @@ class FeedThread(Thread):
self.content_type = feed_type
def do(self, entries, export_path, relative_origin):
notify(getattr(messages, "generating_"+self.content_type))
notify("│\t └─ "+getattr(messages, "generating_"+self.content_type))
self.export_path = export_path
self.relative_origin = relative_origin
self.organize_entries(entries)
......
......@@ -18,7 +18,6 @@
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
from venc2.threads import Thread
from venc2.threads.feed import FeedThread
class MainThread(Thread):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
......@@ -41,32 +40,39 @@ class MainThread(Thread):
self.relative_origin = str()
self.export_path = "blog/"
self.in_thread = True
disable_rss_feed = self.datastore.blog_configuration["disable_rss_feed"]
disable_atom_feed = self.datastore.blog_configuration["disable_atom_feed"]
entries = []
if (not disable_rss_feed) or (not disable_atom_feed):
entries += self.get_feed_entries()
from venc2.threads.feed import FeedThread
if not self.datastore.blog_configuration["disable_rss_feed"]:
self.rss_feed = FeedThread(datastore, theme, patterns, forbidden, "rss")
if not self.datastore.blog_configuration["disable_atom_feed"]:
self.atom_feed = FeedThread(datastore, theme, patterns, forbidden, "atom")
def do(self):
super().do()
i = 0
if not disable_atom_feed:
FeedThread(datastore, theme, patterns, forbidden, "atom").do(entries, self.export_path, self.relative_origin)
if not disable_rss_feed:
FeedThread(datastore, theme, patterns, forbidden, "rss").do(entries, self.export_path, self.relative_origin)
def get_feed_entries(self):
entries = []
i = 0
for entry in self.datastore.get_entries(True):
entries.append(entry)
i+=1
if i == self.datastore.blog_configuration["feed_lenght"]:
break
if not self.datastore.blog_configuration["disable_rss_feed"]:
self.rss_feed.do(entries, self.export_path, self.relative_origin)
if not self.datastore.blog_configuration["disable_atom_feed"]:
self.atom_feed.do(entries, self.export_path, self.relative_origin)
return entries
def do(self):
super().do()
if self.datastore.enable_jsonld:
import json
dump = json.dumps(self.datastore.root_as_jsonld)
f = open("blog/root.jsonld", 'w')
f.write(dump)
def JSONLD(self, argv):
if self.current_page == 0:
return '<script type="application/ld+json" src="blog.jsonld"></script>'
return '<script type="application/ld+json" src="root.jsonld"></script>'
return ''
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment