Commit 05c43a1f authored by Denis Salem's avatar Denis Salem
Browse files

WIP on JSON-LD generation

parent 8b6d540d
......@@ -140,7 +140,7 @@
DONE | Add support for Atom feed.
DONE | Add blog generation timestamp.
DONE | Split unit-test from main branch
DONE | Support ForPage in entries thread.
DONE | Support ForPage in entries thread.
DONE | Update command help with.
DONE | Handle ftp encoding error.
DONE | Sort by metadata.
......@@ -155,6 +155,7 @@
DONE | fix FAQ categories separator is ' > ', not '>'
DONE | fix FAQ subfolders must not start with '/'
DONE | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
TODO | Add definition for optional fields in blog configuration .
TODO | Open with navigator manual.
TODO | Client-side search engine.
TODO | Reorganize themes dependencies in setup (adding default templates, scripts, and pages to include).
......@@ -162,6 +163,7 @@
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
# version 2.x.x
TODO | Add support for access metadata by key.
TODO | Make some dependencies optional.
TODO | Split themes from main branch
TODO | Add translation for Deutsch and Spanish.
......@@ -179,3 +181,11 @@
TODO | Handle missing args in case of unknown number of args (Pattern).
TODO | Add configuration field type check.
TODO | Optimize modules importation.
TODO | Concatenate user defined blog keywords with entries keywords.
TODO | Add more informations into JSON-LD document.
TODO | Optimize JSON-LD processing.
TODO | Embed as much as possible microdata/JSON-LD into blog.
# version 3.x.x
TODO | Add support for plugin.
TODO | Add Incremental blog update.
......@@ -98,10 +98,11 @@ non_contextual_pattern_names_datastore = {
non_contextual_pattern_names_ml = {
"CodeHighlight" : code_highlight.highlight,
"Latex2MathML" : Latex2MathML
"Latex2MathML" : Latex2MathML,
}
contextual_pattern_names = {
"JSON-LD" : "JSONLD",
"IfInThread" : "if_in_thread",
"IfInArchives" : "if_in_archives",
"IfInCategories" : "if_in_categories",
......@@ -231,6 +232,13 @@ def export_blog(argv=list()):
code_highlight.export_style_sheets()
copy_recursively("extra/","blog/")
copy_recursively(theme_folder+"assets/","blog/")
if datastore.enable_jsonld:
import json
dump = json.dumps(datastore.blog_as_jsonld)
f = open("blog/blog.jsonld", 'w')
f.write(dump)
def copy_recursively(src, dest):
......
#! /usr/bin/python3
# Copyright 2016, 2018 Denis Salem
# Copyright 2016, 2019 Denis Salem
#
# This file is part of VenC.
#
......
#! /usr/bin/python3
# Copyright 2016, 2018 Denis Salem
# Copyright 2016, 2019 Denis Salem
#
# This file is part of VenC.
#
......@@ -138,7 +138,7 @@ def new_blog(argv):
"path_encoding": "utf-8",
"server_port": 8888,
"sort_by":"id",
"enable_rdf":True}
"enable_jsonld":True}
for folder_name in argv:
try:
os.mkdir(folder_name)
......
#! /usr/bin/python3
# Copyright 2016, 2018 Denis Salem
# Copyright 2016, 2019 Denis Salem
#
# This file is part of VenC.
#
......
#! /usr/bin/python3
# Copyright 2016, 2019 Denis Salem
#
# This file is part of VenC.
#
# VenC is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# VenC is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
import os
import http.server
import urllib.parse
......
......@@ -18,7 +18,6 @@
# along with VenC. If not, see <http://www.gnu.org/licenses/>.
import hashlib
import json
import os
import datetime
import urllib.parse
......@@ -42,7 +41,7 @@ class DataStore:
notify(messages.loading_data)
self.blog_configuration = get_blog_configuration()
self.sort_by = self.blog_configuration["sort_by"]
self.enable_rdf = self.blog_configuration["enable_rdf"]
self.enable_jsonld = self.blog_configuration["enable_jsonld"]
self.disable_threads = [thread_name.strip() for thread_name in self.blog_configuration["disable_threads"].split(',')]
self.entries = list()
self.entries_per_dates = list()
......@@ -58,17 +57,19 @@ class DataStore:
self.raw_chapters = {}
self.chapters_index = []
self.html_chapters = {}
self.entries_as_jsonld = {}
# Build entries
try:
jsonld_callback = self.entry_to_jsonld_callback if self.enable_jsonld else None
for filename in yield_entries_content():
self.entries.append(Entry(filename, self.blog_configuration["path"], encoding=self.blog_configuration["path_encoding"]))
self.entries.append(Entry(filename, self.blog_configuration["path"], jsonld_callback=jsonld_callback, encoding=self.blog_configuration["path_encoding"]))
except MalformedPatterns as e:
from venc2.helpers import handle_malformed_patterns
handle_malformed_patterns(e)
self.entries = sorted(self.entries, key = lambda entry : self.sort_by(entry))
self.entries = sorted(self.entries, key = lambda entry : self.sort(entry))
for entry_index in range(0, len(self.entries)):
current_entry = self.entries[entry_index]
......@@ -148,8 +149,66 @@ class DataStore:
"weight": node.weight
})
def dseeker_get_metadata_from_entry_callback(self, metadata):
if self.
# Build JSON-LD doc if any
if self.enable_jsonld:
self.root_site_to_jsonld()
def root_site_to_jsonld(self):
if "https://schema.org" in self.blog_configuration.keys():
optionals = self.blog_configuration["https://schema.org"]
else:
optionals = {}
self.blog_as_jsonld = {
"@context": "http://schema.org",
"@type": ["Blog","WebPage"],
"name": self.blog_configuration["blog_name"],
"url": self.blog_configuration["blog_url"],
"description": self.blog_configuration["blog_description"],
"author": {
"@type" : "Person",
"email" : self.blog_configuration["author_email"],
"description" : self.blog_configuration["author_description"],
"name" : self.blog_configuration["author_name"]
},
"keywords" : self.blog_configuration["blog_keywords"],
"inLanguage" : self.blog_configuration["blog_language"],
"license" : {
"@type": "CreativeWork",
"name": self.blog_configuration["license"]
},
"breadcrumb" : [],
"blogPost" : [],
**optionals
}
def entry_to_jsonld_callback(self, entry):
if hasattr(entry, "schemadotorg"):
optionals = entry.schemadotorg
else:
optionals = {}:
doc = {
"@context": "http://schema.org",
"@type" : ["BlogPosting","WebPage"],
"@id" : self.blog_configuration["blog_url"]+"/entry"+str(entry.id)+".jsonld",
"keywords" : entry.raw_metadata["tags"],
"headline" : entry.title,
"name" : entry.title,
"date" : entry.date.isoformat(),
"inLanguage" : self.blog_configuration["blog_language"],
"author" : [{"name":author["value"], "@type": "Person"} for author in entry.authors],
"url" : entry.url.replace(".:GetRelativeOrigin:.", self.blog_configuration["blog_url"]+"/"),
"breadcrumb" : self.blog_as_jsonld["breadcrumb"],
"relatedLink" : [ {
"@type":"Url"
} for c in]
**optionals
}
self.entries_as_jsonld[entry.id] = doc
def get_chapters(self, argv):
key = ''.join(argv)
......@@ -200,7 +259,7 @@ class DataStore:
else:
self.raw_chapters[chapter] = (entry.id, entry.title, entry.url)
def sort_by(self, entry):
def sort(self, entry):
try:
return getattr(entry, self.sort_by)
......@@ -477,8 +536,3 @@ class DataStore:
self.embed_providers["oembed"][p["provider_url"]].append(e["url"])
return get_embed_content(self.embed_providers, argv)
......@@ -57,7 +57,7 @@ def get_blog_configuration():
"disable_rss_feed",
"disable_atom_feed",
"sort_by",
"enable_dseeker"
"enable_jsonld"
]
everything_is_okay = True
......@@ -84,16 +84,15 @@ def get_blog_configuration():
everything_is_okay = False
notify(messages.missing_mandatory_field_in_blog_conf.format(field),"RED")
try:
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blog_configuration.yaml"),"RED")
if not blog_configuration["markup_language"] in ["none", "Markdown", "reStructuredText"]:
everything_is_okay = False
notify(messages.unknown_markup_language.format(blog_configuration["markup_language"], "blog_configuration.yaml"),"RED")
if blog_configuration["sort_by"] in ['', None]:
blog_configuration["sort_by"] = "id"
if blog_configuration["sort_by"] in ['', None]:
blog_configuration["sort_by"] = "id"
except KeyError:
pass
if blog_configuration["blog_url"][-1:] == '/':
blog_configuration["blog_url"] = blog_configuration["blog_url"][:-1]
if not everything_is_okay:
exit()
......
......@@ -56,7 +56,7 @@ class EntryWrapper:
die(messages.missing_entry_content_inclusion)
class Entry:
def __init__(self, filename, paths, encoding="utf-8"):
def __init__(self, filename, paths, jsonld_callback, encoding="utf-8", ):
self.previous_entry = None
self.next_entry = None
......@@ -88,7 +88,10 @@ class Entry:
if not key in ["authors", "tags", "categories", "title"]:
if metadata[key] != None:
setattr(self, key, metadata[key])
elif key == "https://schema.org" and metadata[key] != None:
setattr(self, "schemadotorg", metadata[key]
else:
notify(messages.invalid_or_missing_metadata.format(key, filename), color="YELLOW")
setattr(self, key, '')
......@@ -99,6 +102,7 @@ class Entry:
notify(messages.invalid_or_missing_metadata.format(key, filename), color="YELLOW")
metadata[key] = ''
self.raw_metadata = metadata
self.filename = filename
self.id = int(filename.split('__')[0])
......@@ -129,10 +133,6 @@ class Entry:
except KeyError:
die(messages.missing_mandatory_field_in_entry.format("tags", self.id))
except Exception as e:
print(self.id, metadata)
raise e
params = {
"entry_id": self.id,
"entry_title": self.title
......@@ -158,7 +158,8 @@ class Entry:
self.categories_leaves.append({
"value": category_leaf,
"path":category_leaf_url
"path":category_leaf_url,
"raw" : category
})
except IndexError : # when list is empty
......@@ -170,6 +171,8 @@ class Entry:
self.html_tags = {}
self.html_authors = {}
self.html_categories_leaves = {}
if jsonld_callback != None:
jsonld_callback(self)
''' Iterate through entries folder '''
def yield_entries_content():
......
......@@ -154,6 +154,9 @@ class Thread:
return output[:-len(separator)]
def JSONLD(self, argv):
return ''
def if_in_first_page(self, argv):
if self.current_page == 0:
return argv[0].strip()
......
......@@ -64,6 +64,11 @@ class MainThread(Thread):
if not self.datastore.blog_configuration["disable_atom_feed"]:
self.atom_feed.do(entries, self.export_path, self.relative_origin)
def JSONLD(self, argv):
if self.current_page == 0:
return '<script type="application/ld+json" src="blog.jsonld"></script>'
return ''
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment