Commit 6580af1e authored by Denis Salem's avatar Denis Salem

wip on JSON-LD for archives

parent 3533b93d
......@@ -156,7 +156,7 @@
DONE | fix FAQ subfolders must not start with '/'
DONE | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
DONE | Refactor DatesThread to ArchivesThread.
TODO | Add definition for optional fields in blog configuration.
TODO | Add definition in documentation for optional fields in blog configuration.
TODO | Open with navigator manual.
WIP | Client-side search engine.
WIP | Semantic-web features.
......@@ -164,7 +164,8 @@
TODO | Add relevant templates for themes.
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
# version 2.x.x
# version 2.1.x
TODO | Create entry with default template, if defined.
TODO | Add support for access metadata by key.
TODO | Make some dependencies optional.
TODO | Split themes from main branch
......@@ -185,12 +186,14 @@
TODO | Optimize modules importation.
TODO | Concatenate user defined blog keywords with entries keywords.
TODO | Add more informations into JSON-LD document.
TODO | Optimize JSON-LD processing.
TODO | Optimize and refine JSON-LD processing.
TODO | Embed as much as possible microdata/JSON-LD into blog.
TODO | Automatically add entries categories to entries tags.
TODO | Make entry URL nicer when filename is index.html and has subfolder.
TODO | Catch "OSError: [Errno 98] Address already in use".
# version 3.x.x
TODO | Add support for plugin.
TODO | Add Incremental blog update.
TODO | Add support for blog within blog.
TODO | Add support for single page.
TODO | Add support for sub-blogs within main blog.
TODO | Add support for single pages.
......@@ -60,14 +60,27 @@ class DataStore:
# Build JSON-LD doc if any
if self.enable_jsonld:
if "https://schema.org" in self.blog_configuration.keys():
self.optionals_schemadotorg = self.blog_configuration["https://schema.org"]
else:
self.optionals_schemadotorg = {}
self.entries_as_jsonld = {}
self.archives_as_jsonld = {}
self.root_site_to_jsonld()
# Build entries
try:
jsonld_callback = self.entry_to_jsonld_callback if self.enable_jsonld else None
for filename in yield_entries_content():
self.entries.append(Entry(filename, self.blog_configuration["path"], jsonld_callback=jsonld_callback, encoding=self.blog_configuration["path_encoding"]))
self.entries.append(Entry(
filename,
self.blog_configuration["path"],
jsonld_callback,
self.blog_configuration["path"]["dates_directory_name"],
self.blog_configuration["path_encoding"]
))
except MalformedPatterns as e:
from venc2.helpers import handle_malformed_patterns
......@@ -83,11 +96,11 @@ class DataStore:
# Update entriesPerDates
if self.blog_configuration["path"]["dates_directory_name"] != '':
formatted_date = current_entry.date.strftime(self.blog_configuration["path"]["dates_directory_name"])
entries_indexes = self.get_entries_index_for_given_date(formatted_date)
if entries_indexes != None:
self.entries_per_dates[entries_indexes].count +=1
self.entries_per_dates[entries_indexes].related_to.append(entry_index)
formatted_date = current_entry.formatted_date
entries_index = self.get_entries_index_for_given_date(formatted_date)
if entries_index != None:
self.entries_per_dates[entries_index].count +=1
self.entries_per_dates[entries_index].related_to.append(entry_index)
else:
self.entries_per_dates.append(MetadataNode(formatted_date, entry_index))
......@@ -134,7 +147,7 @@ class DataStore:
top = top[-1].sub_chapters
# Setup BlogDates Data
# Setup BlogArchives Data
self.blog_dates = list()
for node in self.entries_per_dates:
try:
......@@ -152,14 +165,13 @@ class DataStore:
"count": node.count,
"weight": node.weight
})
# Setup archives as jsonld if any
if self.enable_jsonld:
for archive in self.entries_per_dates:
self.archives_to_jsonld(archive.value, archive.related_to)
def root_site_to_jsonld(self):
if "https://schema.org" in self.blog_configuration.keys():
optionals = self.blog_configuration["https://schema.org"]
else:
optionals = {}
self.root_as_jsonld = {
"@context": "http://schema.org",
"@type": ["Blog","WebPage"],
......@@ -180,9 +192,33 @@ class DataStore:
"name": self.blog_configuration["license"]
},
"blogPost" : [],
**optionals
**self.optionals_schemadotorg
}
def archives_to_jsonld(self, archive_value, related_to):
self.archives_as_jsonld[archive_value] = {
"@context": "http://schema.org",
"@type": ["Blog","WebPage"],
"@id" : self.blog_configuration["blog_url"]+'/'+archive_value+"/archives.jsonld",
"name": self.blog_configuration["blog_name"],
"url": self.blog_configuration["blog_url"]+'/'+archive_value,
"description": self.blog_configuration["blog_description"],
"author": {
"@type" : "Person",
"email" : self.blog_configuration["author_email"],
"description" : self.blog_configuration["author_description"],
"name" : self.blog_configuration["author_name"]
},
"keywords" : self.blog_configuration["blog_keywords"],
"inLanguage" : self.blog_configuration["blog_language"],
"license" : {
"@type": "CreativeWork",
"name": self.blog_configuration["license"]
},
"blogPost" : [],
**self.optionals_schemadotorg
}
def entry_to_jsonld_callback(self, entry):
if hasattr(entry, "schemadotorg"):
optionals = entry.schemadotorg
......@@ -190,16 +226,34 @@ class DataStore:
else:
optionals = {}
author = [{"name":author["value"], "@type": "Person"} for author in entry.authors]
if "publisher" in entry.schemadotorg.keys():
publisher = entry.schemadotorg["publisher"]
elif "publisher" in self.blog_configuration["https://schema.org"].keys():
publisher = self.blog_configuration["https://schema.org"]["publisher"]
elif author != []:
publisher = author
else:
publisher = {
"@type":"Person",
"name":self.blog_configuration["author_name"]
}
entry_url = '/'.join(entry.url.split('/')[:-1]).replace(".:GetRelativeOrigin:.", self.blog_configuration["blog_url"]+'/')
doc = {
"@context": "http://schema.org",
"@type" : ["BlogPosting", "WebPage"],
"@id" : self.blog_configuration["blog_url"]+"/entry"+str(entry.id)+".jsonld",
"keywords" : entry.raw_metadata["tags"],
"@id" : entry_url+"/entry"+str(entry.id)+".jsonld",
"keywords" : entry.raw_metadata["tags"]+", "+entry.raw_metadata["categories"],
"headline" : entry.title,
"name" : entry.title,
"date" : entry.date.isoformat(),
"datePublished" : entry.date.isoformat(),
"inLanguage" : self.blog_configuration["blog_language"],
"author" : [{"name":author["value"], "@type": "Person"} for author in entry.authors],
"author" : author if author != [] else self.blog_configuration["author_name"],
"publisher" : publisher,
"url" : entry.url.replace(".:GetRelativeOrigin:.", self.blog_configuration["blog_url"]+"/"),
"breadcrumb" : {
"itemListElement": [
......@@ -208,8 +262,21 @@ class DataStore:
"relatedLink" : [ c["path"] for c in entry.categories_leaves],
**optionals
}
self.entries_as_jsonld[entry.id] = doc
# TODO 2.x.x : TRY AVOID DEREFERENCE HERE
blogPost = {
"@type": doc["@type"],
"@id": doc["@id"],
"headline":entry.title,
"author": doc["author"],
"publisher": doc["publisher"],
"datePublished": doc["datePublished"],
"keywords": doc["keywords"],
"url": doc["url"]
}
self.root_as_jsonld["blogPost"].append(blog_post)
self.archives_as_jsonld[entry.formatted_date]["blogPost"].append(blog_post)
def get_chapters(self, argv):
key = ''.join(argv)
......
#! /usr/bin/python3
# Copyright 2016, 2017 Denis Salem
# Copyright 2016, 2019 Denis Salem
# This file is part of VenC.
#
......@@ -32,18 +32,18 @@ def get_blog_configuration():
"blog_name",
"text_editor",
"date_format",
"author_name",
"blog_description",
"blog_keywords",
"author_description",
"license",
"blog_url",
"author_name",
"blog_description",
"blog_keywords",
"author_description",
"license",
"blog_url",
"ftp_host",
"blog_language",
"author_email",
"entries_per_pages",
"blog_language",
"author_email",
"entries_per_pages",
"columns",
"feed_lenght",
"feed_lenght",
"reverse_thread_order",
"markup_language",
"disable_threads",
......@@ -68,11 +68,11 @@ def get_blog_configuration():
mandatory_fields = [
"index_file_name",
"category_directory_name",
"dates_directory_name",
"entry_file_name",
"rss_file_name",
"atom_file_name",
"category_directory_name",
"dates_directory_name",
"entry_file_name",
"rss_file_name",
"atom_file_name",
"ftp",
"entries_sub_folders",
"categories_sub_folders",
......@@ -84,6 +84,9 @@ def get_blog_configuration():
everything_is_okay = False
notify(messages.missing_mandatory_field_in_blog_conf.format(field),"RED")
if not "https://schema.org" in blog_configuration.keys():
blog_configuration["https://schema.org"] = {}
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blog_configuration.yaml"),"RED")
......
......@@ -56,9 +56,10 @@ class EntryWrapper:
die(messages.missing_entry_content_inclusion)
class Entry:
def __init__(self, filename, paths, jsonld_callback, encoding="utf-8", ):
def __init__(self, filename, paths, jsonld_callback, date_format, encoding="utf-8", ):
self.previous_entry = None
self.next_entry = None
self.schemadotorg = {}
# Loading
raw_data = open(os.getcwd()+"/entries/"+filename,'r').read()
......@@ -90,7 +91,7 @@ class Entry:
setattr(self, key, metadata[key])
elif key == "https://schema.org" and metadata[key] != None:
setattr(self, "schemadotorg", metadata[key])
self.schemadotorg = metadata[key]
else:
notify(messages.invalid_or_missing_metadata.format(key, filename), color="YELLOW")
......@@ -114,7 +115,8 @@ class Entry:
hour=int(raw_date[3]),
minute=int(raw_date[4])
)
self.formatted_date = self.date.strftime(date_format)
try:
self.title = metadata["title"].replace(".:GetEntryTitle:.",'')
......
#! /usr/bin/python3
# Copyright 2016, 2018 Denis Salem
# Copyright 2016, 2019 Denis Salem
#
# This file is part of VenC.
#
......
......@@ -103,3 +103,6 @@ class Messages:
malformed_escape_patterns_missing_opening_symbols = "Malformed escaping patterns in \"{0}\": One or more opening symbols are missing."
malformed_escape_patterns_missing_closing_symbols = "Malformed escaping patterns in \"{0}\": One or more closing symbols are missing."
copy_assets_and_extra_files = "Copying extra files..."
generating_jsonld_doc = "Generating JSON-LD document..."
generating_jsonld_docs = "Generating JSON-LD documents..."
......@@ -101,4 +101,6 @@ class Messages:
malformed_escape_patterns_missing_opening_symbols = "Motifs d'échappement mal formés dans \"{0}\": Il manque un ou plusieurs symboles d'ouverture."
malformed_escape_patterns_missing_closing_symbols = "Motifs d'échappement mal formés dans \"{0}\": Il manque un ou plusieurs symboles de fermeture."
copy_assets_and_extra_files = "Copie des fichiers additionelles..."
generating_jsonld_doc = "Génération du document JSON-LD..."
generating_jsonld_docs = "Génération des documents JSON-LD..."
......@@ -29,7 +29,9 @@ from venc2.patterns.processor import Processor
class Thread:
def __init__(self, prompt, datastore, theme, patterns, forbidden):
self.indentation_level = "│ "
self.datastore = datastore
self.patterns = patterns
# Notify wich thread is processed
if prompt != "":
notify("├─ "+prompt)
......
......@@ -26,7 +26,6 @@ from venc2.threads import Thread
class CategoriesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
super().__init__(prompt, datastore, theme, patterns, forbidden)
self.indentation_level = "│ "
self.filename = self.datastore.blog_configuration["path"]["index_file_name"]
self.export_path = "blog/"+self.datastore.blog_configuration["path"]["categories_sub_folders"]+'/'
self.relative_origin = ""
......
......@@ -22,6 +22,8 @@ import os
from venc2.prompt import notify
from venc2.threads import Thread
from venc2.l10n import messages
import json
class EntriesThread(Thread):
def __init__(self, prompt, datastore, theme, patterns, forbidden):
......@@ -120,16 +122,12 @@ class EntriesThread(Thread):
)
stream.write(output)
stream.close()
def iterate_through_pages(self):
for page in self.pages:
for entry in page:
self.pre_iteration()
self.setup_context(entry)
self.do_iteration(entry)
self.post_iteration()
self.current_export_path = export_path
def do(self):
if self.datastore.enable_jsonld:
notify(self.indentation_level+' └─ '+messages.generating_jsonld_docs)
self.page_number = 0
self.current_page = 0
if len(self.pages):
......@@ -139,10 +137,14 @@ class EntriesThread(Thread):
self.pre_iteration()
self.do_iteration(entry)
self.post_iteration()
if self.datastore.enable_jsonld:
dump = json.dumps(self.datastore.entries_as_jsonld[self.current_entry.id])
f = open(self.current_export_path+"/entry"+str(entry.id)+".jsonld", 'w')
f.write(dump)
def JSONLD(self, argv):
if self.current_page == 0:
if self.datastore.enable_jsonld:
return '<script type="application/ld+json" src="entry'+str(self.current_entry.id)+'.jsonld"></script>'
return ''
......@@ -40,18 +40,6 @@ class MainThread(Thread):
self.relative_origin = str()
self.export_path = "blog/"
self.in_thread = True
disable_rss_feed = self.datastore.blog_configuration["disable_rss_feed"]
disable_atom_feed = self.datastore.blog_configuration["disable_atom_feed"]
entries = []
if (not disable_rss_feed) or (not disable_atom_feed):
entries += self.get_feed_entries()
from venc2.threads.feed import FeedThread
if not disable_atom_feed:
FeedThread(datastore, theme, patterns, forbidden, "atom").do(entries, self.export_path, self.relative_origin, "│ ", '└' if disable_atom_feed else '├')
if not disable_rss_feed:
FeedThread(datastore, theme, patterns, forbidden, "rss").do(entries, self.export_path, self.relative_origin, "│ ", '└')
def get_feed_entries(self):
entries = []
......@@ -64,14 +52,30 @@ class MainThread(Thread):
def do(self):
super().do()
if self.datastore.enable_jsonld:
disable_rss_feed = self.datastore.blog_configuration["disable_rss_feed"]
disable_atom_feed = self.datastore.blog_configuration["disable_atom_feed"]
entries = []
if (not disable_rss_feed) or (not disable_atom_feed):
entries += self.get_feed_entries()
from venc2.threads.feed import FeedThread
if not disable_atom_feed:
FeedThread(self.datastore, self.theme, self.patterns, self.forbidden, "atom").do(entries, self.export_path, self.relative_origin, "│ ", '└' if disable_atom_feed and not self.datastore.enable_jsonld else '├')
if not disable_rss_feed:
FeedThread(self.datastore, self.theme, self.patterns, self.forbidden, "rss").do(entries, self.export_path, self.relative_origin, "│ ", '└' if not self.datastore.enable_jsonld else '├')
if self.datastore.enable_jsonld:
from venc2.prompt import notify
from venc2.l10n import messages
import json
notify(self.indentation_level+' └─ '+messages.generating_jsonld_doc)
dump = json.dumps(self.datastore.root_as_jsonld)
f = open("blog/root.jsonld", 'w')
f.write(dump)
def JSONLD(self, argv):
if self.current_page == 0:
if self.current_page == 0 and self.datastore.enable_jsonld:
return '<script type="application/ld+json" src="root.jsonld"></script>'
return ''
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment