Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
Denis Salem
VenC
Commits
02a98632
Commit
02a98632
authored
Feb 19, 2019
by
Denis Salem
Browse files
wip on JSON-LD for archives
parent
3a47cd88
Changes
11
Hide whitespace changes
Inline
Side-by-side
changelog.md
View file @
02a98632
...
...
@@ -156,7 +156,7 @@
DONE | fix FAQ subfolders must not start with '/'
DONE | fix FAQ https://stackoverflow.com/questions/14547631/python-locale-error-unsupported-locale-setting.
DONE | Refactor DatesThread to ArchivesThread.
TODO | Add definition for optional fields in blog configuration.
TODO | Add definition
in documentation
for optional fields in blog configuration.
TODO | Open with navigator manual.
WIP | Client-side search engine.
WIP | Semantic-web features.
...
...
@@ -164,7 +164,8 @@
TODO | Add relevant templates for themes.
WIP | Change nomenclature: Most of the usual patterns are changed. Fix doc.
# version 2.x.x
# version 2.1.x
TODO | Create entry with default template, if defined.
TODO | Add support for access metadata by key.
TODO | Make some dependencies optional.
TODO | Split themes from main branch
...
...
@@ -185,12 +186,14 @@
TODO | Optimize modules importation.
TODO | Concatenate user defined blog keywords with entries keywords.
TODO | Add more informations into JSON-LD document.
TODO | Optimize JSON-LD processing.
TODO | Optimize
and refine
JSON-LD processing.
TODO | Embed as much as possible microdata/JSON-LD into blog.
TODO | Automatically add entries categories to entries tags.
TODO | Make entry URL nicer when filename is index.html and has subfolder.
TODO | Catch "OSError: [Errno 98] Address already in use".
# version 3.x.x
TODO | Add support for plugin.
TODO | Add Incremental blog update.
TODO | Add support for blog within blog.
TODO | Add support for single page.
TODO | Add support for
sub-
blog
s
within
main
blog.
TODO | Add support for single page
s
.
src/venc2/datastore/__init__.py
View file @
02a98632
...
...
@@ -60,14 +60,27 @@ class DataStore:
# Build JSON-LD doc if any
if
self
.
enable_jsonld
:
if
"https://schema.org"
in
self
.
blog_configuration
.
keys
():
self
.
optionals_schemadotorg
=
self
.
blog_configuration
[
"https://schema.org"
]
else
:
self
.
optionals_schemadotorg
=
{}
self
.
entries_as_jsonld
=
{}
self
.
archives_as_jsonld
=
{}
self
.
root_site_to_jsonld
()
# Build entries
try
:
jsonld_callback
=
self
.
entry_to_jsonld_callback
if
self
.
enable_jsonld
else
None
for
filename
in
yield_entries_content
():
self
.
entries
.
append
(
Entry
(
filename
,
self
.
blog_configuration
[
"path"
],
jsonld_callback
=
jsonld_callback
,
encoding
=
self
.
blog_configuration
[
"path_encoding"
]))
self
.
entries
.
append
(
Entry
(
filename
,
self
.
blog_configuration
[
"path"
],
jsonld_callback
,
self
.
blog_configuration
[
"path"
][
"dates_directory_name"
],
self
.
blog_configuration
[
"path_encoding"
]
))
except
MalformedPatterns
as
e
:
from
venc2.helpers
import
handle_malformed_patterns
...
...
@@ -83,11 +96,11 @@ class DataStore:
# Update entriesPerDates
if
self
.
blog_configuration
[
"path"
][
"dates_directory_name"
]
!=
''
:
formatted_date
=
current_entry
.
date
.
strftime
(
self
.
blog_configuration
[
"path"
][
"da
te
s
_d
irectory_name"
])
entries_index
es
=
self
.
get_entries_index_for_given_date
(
formatted_date
)
if
entries_index
es
!=
None
:
self
.
entries_per_dates
[
entries_index
es
].
count
+=
1
self
.
entries_per_dates
[
entries_index
es
].
related_to
.
append
(
entry_index
)
formatted_date
=
current_entry
.
format
te
d
_d
ate
entries_index
=
self
.
get_entries_index_for_given_date
(
formatted_date
)
if
entries_index
!=
None
:
self
.
entries_per_dates
[
entries_index
].
count
+=
1
self
.
entries_per_dates
[
entries_index
].
related_to
.
append
(
entry_index
)
else
:
self
.
entries_per_dates
.
append
(
MetadataNode
(
formatted_date
,
entry_index
))
...
...
@@ -134,7 +147,7 @@ class DataStore:
top
=
top
[
-
1
].
sub_chapters
# Setup Blog
Dat
es Data
# Setup Blog
Archiv
es Data
self
.
blog_dates
=
list
()
for
node
in
self
.
entries_per_dates
:
try
:
...
...
@@ -152,14 +165,13 @@ class DataStore:
"count"
:
node
.
count
,
"weight"
:
node
.
weight
})
# Setup archives as jsonld if any
if
self
.
enable_jsonld
:
for
archive
in
self
.
entries_per_dates
:
self
.
archives_to_jsonld
(
archive
.
value
,
archive
.
related_to
)
def
root_site_to_jsonld
(
self
):
if
"https://schema.org"
in
self
.
blog_configuration
.
keys
():
optionals
=
self
.
blog_configuration
[
"https://schema.org"
]
else
:
optionals
=
{}
self
.
root_as_jsonld
=
{
"@context"
:
"http://schema.org"
,
"@type"
:
[
"Blog"
,
"WebPage"
],
...
...
@@ -180,9 +192,33 @@ class DataStore:
"name"
:
self
.
blog_configuration
[
"license"
]
},
"blogPost"
:
[],
**
optionals
**
self
.
optionals
_schemadotorg
}
def
archives_to_jsonld
(
self
,
archive_value
,
related_to
):
self
.
archives_as_jsonld
[
archive_value
]
=
{
"@context"
:
"http://schema.org"
,
"@type"
:
[
"Blog"
,
"WebPage"
],
"@id"
:
self
.
blog_configuration
[
"blog_url"
]
+
'/'
+
archive_value
+
"/archives.jsonld"
,
"name"
:
self
.
blog_configuration
[
"blog_name"
],
"url"
:
self
.
blog_configuration
[
"blog_url"
]
+
'/'
+
archive_value
,
"description"
:
self
.
blog_configuration
[
"blog_description"
],
"author"
:
{
"@type"
:
"Person"
,
"email"
:
self
.
blog_configuration
[
"author_email"
],
"description"
:
self
.
blog_configuration
[
"author_description"
],
"name"
:
self
.
blog_configuration
[
"author_name"
]
},
"keywords"
:
self
.
blog_configuration
[
"blog_keywords"
],
"inLanguage"
:
self
.
blog_configuration
[
"blog_language"
],
"license"
:
{
"@type"
:
"CreativeWork"
,
"name"
:
self
.
blog_configuration
[
"license"
]
},
"blogPost"
:
[],
**
self
.
optionals_schemadotorg
}
def
entry_to_jsonld_callback
(
self
,
entry
):
if
hasattr
(
entry
,
"schemadotorg"
):
optionals
=
entry
.
schemadotorg
...
...
@@ -190,16 +226,34 @@ class DataStore:
else
:
optionals
=
{}
author
=
[{
"name"
:
author
[
"value"
],
"@type"
:
"Person"
}
for
author
in
entry
.
authors
]
if
"publisher"
in
entry
.
schemadotorg
.
keys
():
publisher
=
entry
.
schemadotorg
[
"publisher"
]
elif
"publisher"
in
self
.
blog_configuration
[
"https://schema.org"
].
keys
():
publisher
=
self
.
blog_configuration
[
"https://schema.org"
][
"publisher"
]
elif
author
!=
[]:
publisher
=
author
else
:
publisher
=
{
"@type"
:
"Person"
,
"name"
:
self
.
blog_configuration
[
"author_name"
]
}
entry_url
=
'/'
.
join
(
entry
.
url
.
split
(
'/'
)[:
-
1
]).
replace
(
".:GetRelativeOrigin:."
,
self
.
blog_configuration
[
"blog_url"
]
+
'/'
)
doc
=
{
"@context"
:
"http://schema.org"
,
"@type"
:
[
"BlogPosting"
,
"WebPage"
],
"@id"
:
self
.
blog_configuration
[
"blog
_url
"
]
+
"/entry"
+
str
(
entry
.
id
)
+
".jsonld"
,
"keywords"
:
entry
.
raw_metadata
[
"tags"
],
"@id"
:
entry
_url
+
"/entry"
+
str
(
entry
.
id
)
+
".jsonld"
,
"keywords"
:
entry
.
raw_metadata
[
"tags"
]
+
", "
+
entry
.
raw_metadata
[
"categories"
]
,
"headline"
:
entry
.
title
,
"name"
:
entry
.
title
,
"date"
:
entry
.
date
.
isoformat
(),
"date
Published
"
:
entry
.
date
.
isoformat
(),
"inLanguage"
:
self
.
blog_configuration
[
"blog_language"
],
"author"
:
[{
"name"
:
author
[
"value"
],
"@type"
:
"Person"
}
for
author
in
entry
.
authors
],
"author"
:
author
if
author
!=
[]
else
self
.
blog_configuration
[
"author_name"
],
"publisher"
:
publisher
,
"url"
:
entry
.
url
.
replace
(
".:GetRelativeOrigin:."
,
self
.
blog_configuration
[
"blog_url"
]
+
"/"
),
"breadcrumb"
:
{
"itemListElement"
:
[
...
...
@@ -208,8 +262,21 @@ class DataStore:
"relatedLink"
:
[
c
[
"path"
]
for
c
in
entry
.
categories_leaves
],
**
optionals
}
self
.
entries_as_jsonld
[
entry
.
id
]
=
doc
# TODO 2.x.x : TRY AVOID DEREFERENCE HERE
blogPost
=
{
"@type"
:
doc
[
"@type"
],
"@id"
:
doc
[
"@id"
],
"headline"
:
entry
.
title
,
"author"
:
doc
[
"author"
],
"publisher"
:
doc
[
"publisher"
],
"datePublished"
:
doc
[
"datePublished"
],
"keywords"
:
doc
[
"keywords"
],
"url"
:
doc
[
"url"
]
}
self
.
root_as_jsonld
[
"blogPost"
].
append
(
blog_post
)
self
.
archives_as_jsonld
[
entry
.
formatted_date
][
"blogPost"
].
append
(
blog_post
)
def
get_chapters
(
self
,
argv
):
key
=
''
.
join
(
argv
)
...
...
src/venc2/datastore/configuration.py
View file @
02a98632
#! /usr/bin/python3
# Copyright 2016, 201
7
Denis Salem
# Copyright 2016, 201
9
Denis Salem
# This file is part of VenC.
#
...
...
@@ -32,18 +32,18 @@ def get_blog_configuration():
"blog_name"
,
"text_editor"
,
"date_format"
,
"author_name"
,
"blog_description"
,
"blog_keywords"
,
"author_description"
,
"license"
,
"blog_url"
,
"author_name"
,
"blog_description"
,
"blog_keywords"
,
"author_description"
,
"license"
,
"blog_url"
,
"ftp_host"
,
"blog_language"
,
"author_email"
,
"entries_per_pages"
,
"blog_language"
,
"author_email"
,
"entries_per_pages"
,
"columns"
,
"feed_lenght"
,
"feed_lenght"
,
"reverse_thread_order"
,
"markup_language"
,
"disable_threads"
,
...
...
@@ -68,11 +68,11 @@ def get_blog_configuration():
mandatory_fields
=
[
"index_file_name"
,
"category_directory_name"
,
"dates_directory_name"
,
"entry_file_name"
,
"rss_file_name"
,
"atom_file_name"
,
"category_directory_name"
,
"dates_directory_name"
,
"entry_file_name"
,
"rss_file_name"
,
"atom_file_name"
,
"ftp"
,
"entries_sub_folders"
,
"categories_sub_folders"
,
...
...
@@ -84,6 +84,9 @@ def get_blog_configuration():
everything_is_okay
=
False
notify
(
messages
.
missing_mandatory_field_in_blog_conf
.
format
(
field
),
"RED"
)
if
not
"https://schema.org"
in
blog_configuration
.
keys
():
blog_configuration
[
"https://schema.org"
]
=
{}
if
not
blog_configuration
[
"markup_language"
]
in
[
"none"
,
"Markdown"
,
"reStructuredText"
]:
everything_is_okay
=
False
notify
(
messages
.
unknown_markup_language
.
format
(
blog_configuration
[
"markup_language"
],
"blog_configuration.yaml"
),
"RED"
)
...
...
src/venc2/datastore/entry.py
View file @
02a98632
...
...
@@ -56,9 +56,10 @@ class EntryWrapper:
die
(
messages
.
missing_entry_content_inclusion
)
class
Entry
:
def
__init__
(
self
,
filename
,
paths
,
jsonld_callback
,
encoding
=
"utf-8"
,
):
def
__init__
(
self
,
filename
,
paths
,
jsonld_callback
,
date_format
,
encoding
=
"utf-8"
,
):
self
.
previous_entry
=
None
self
.
next_entry
=
None
self
.
schemadotorg
=
{}
# Loading
raw_data
=
open
(
os
.
getcwd
()
+
"/entries/"
+
filename
,
'r'
).
read
()
...
...
@@ -90,7 +91,7 @@ class Entry:
setattr
(
self
,
key
,
metadata
[
key
])
elif
key
==
"https://schema.org"
and
metadata
[
key
]
!=
None
:
se
tattr
(
self
,
"
schemadotorg
"
,
metadata
[
key
]
)
se
lf
.
schemadotorg
=
metadata
[
key
]
else
:
notify
(
messages
.
invalid_or_missing_metadata
.
format
(
key
,
filename
),
color
=
"YELLOW"
)
...
...
@@ -114,7 +115,8 @@ class Entry:
hour
=
int
(
raw_date
[
3
]),
minute
=
int
(
raw_date
[
4
])
)
self
.
formatted_date
=
self
.
date
.
strftime
(
date_format
)
try
:
self
.
title
=
metadata
[
"title"
].
replace
(
".:GetEntryTitle:."
,
''
)
...
...
src/venc2/l10n/__init__.py
View file @
02a98632
#! /usr/bin/python3
# Copyright 2016, 201
8
Denis Salem
# Copyright 2016, 201
9
Denis Salem
#
# This file is part of VenC.
#
...
...
src/venc2/l10n/en.py
View file @
02a98632
...
...
@@ -103,3 +103,6 @@ class Messages:
malformed_escape_patterns_missing_opening_symbols
=
"Malformed escaping patterns in
\"
{0}
\"
: One or more opening symbols are missing."
malformed_escape_patterns_missing_closing_symbols
=
"Malformed escaping patterns in
\"
{0}
\"
: One or more closing symbols are missing."
copy_assets_and_extra_files
=
"Copying extra files..."
generating_jsonld_doc
=
"Generating JSON-LD document..."
generating_jsonld_docs
=
"Generating JSON-LD documents..."
src/venc2/l10n/fr.py
View file @
02a98632
...
...
@@ -101,4 +101,6 @@ class Messages:
malformed_escape_patterns_missing_opening_symbols
=
"Motifs d'échappement mal formés dans
\"
{0}
\"
: Il manque un ou plusieurs symboles d'ouverture."
malformed_escape_patterns_missing_closing_symbols
=
"Motifs d'échappement mal formés dans
\"
{0}
\"
: Il manque un ou plusieurs symboles de fermeture."
copy_assets_and_extra_files
=
"Copie des fichiers additionelles..."
generating_jsonld_doc
=
"Génération du document JSON-LD..."
generating_jsonld_docs
=
"Génération des documents JSON-LD..."
src/venc2/threads/__init__.py
View file @
02a98632
...
...
@@ -29,7 +29,9 @@ from venc2.patterns.processor import Processor
class
Thread
:
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
self
.
indentation_level
=
"│ "
self
.
datastore
=
datastore
self
.
patterns
=
patterns
# Notify wich thread is processed
if
prompt
!=
""
:
notify
(
"├─ "
+
prompt
)
...
...
src/venc2/threads/categories.py
View file @
02a98632
...
...
@@ -26,7 +26,6 @@ from venc2.threads import Thread
class
CategoriesThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
super
().
__init__
(
prompt
,
datastore
,
theme
,
patterns
,
forbidden
)
self
.
indentation_level
=
"│ "
self
.
filename
=
self
.
datastore
.
blog_configuration
[
"path"
][
"index_file_name"
]
self
.
export_path
=
"blog/"
+
self
.
datastore
.
blog_configuration
[
"path"
][
"categories_sub_folders"
]
+
'/'
self
.
relative_origin
=
""
...
...
src/venc2/threads/entries.py
View file @
02a98632
...
...
@@ -22,6 +22,8 @@ import os
from
venc2.prompt
import
notify
from
venc2.threads
import
Thread
from
venc2.l10n
import
messages
import
json
class
EntriesThread
(
Thread
):
def
__init__
(
self
,
prompt
,
datastore
,
theme
,
patterns
,
forbidden
):
...
...
@@ -120,16 +122,12 @@ class EntriesThread(Thread):
)
stream
.
write
(
output
)
stream
.
close
()
def
iterate_through_pages
(
self
):
for
page
in
self
.
pages
:
for
entry
in
page
:
self
.
pre_iteration
()
self
.
setup_context
(
entry
)
self
.
do_iteration
(
entry
)
self
.
post_iteration
()
self
.
current_export_path
=
export_path
def
do
(
self
):
if
self
.
datastore
.
enable_jsonld
:
notify
(
self
.
indentation_level
+
' └─ '
+
messages
.
generating_jsonld_docs
)
self
.
page_number
=
0
self
.
current_page
=
0
if
len
(
self
.
pages
):
...
...
@@ -139,10 +137,14 @@ class EntriesThread(Thread):
self
.
pre_iteration
()
self
.
do_iteration
(
entry
)
self
.
post_iteration
()
if
self
.
datastore
.
enable_jsonld
:
dump
=
json
.
dumps
(
self
.
datastore
.
entries_as_jsonld
[
self
.
current_entry
.
id
])
f
=
open
(
self
.
current_export_path
+
"/entry"
+
str
(
entry
.
id
)
+
".jsonld"
,
'w'
)
f
.
write
(
dump
)
def
JSONLD
(
self
,
argv
):
if
self
.
current_page
==
0
:
if
self
.
datastore
.
enable_jsonld
:
return
'<script type="application/ld+json" src="entry'
+
str
(
self
.
current_entry
.
id
)
+
'.jsonld"></script>'
return
''
src/venc2/threads/main.py
View file @
02a98632
...
...
@@ -40,18 +40,6 @@ class MainThread(Thread):
self
.
relative_origin
=
str
()
self
.
export_path
=
"blog/"
self
.
in_thread
=
True
disable_rss_feed
=
self
.
datastore
.
blog_configuration
[
"disable_rss_feed"
]
disable_atom_feed
=
self
.
datastore
.
blog_configuration
[
"disable_atom_feed"
]
entries
=
[]
if
(
not
disable_rss_feed
)
or
(
not
disable_atom_feed
):
entries
+=
self
.
get_feed_entries
()
from
venc2.threads.feed
import
FeedThread
if
not
disable_atom_feed
:
FeedThread
(
datastore
,
theme
,
patterns
,
forbidden
,
"atom"
).
do
(
entries
,
self
.
export_path
,
self
.
relative_origin
,
"│ "
,
'└'
if
disable_atom_feed
else
'├'
)
if
not
disable_rss_feed
:
FeedThread
(
datastore
,
theme
,
patterns
,
forbidden
,
"rss"
).
do
(
entries
,
self
.
export_path
,
self
.
relative_origin
,
"│ "
,
'└'
)
def
get_feed_entries
(
self
):
entries
=
[]
...
...
@@ -64,14 +52,30 @@ class MainThread(Thread):
def
do
(
self
):
super
().
do
()
if
self
.
datastore
.
enable_jsonld
:
disable_rss_feed
=
self
.
datastore
.
blog_configuration
[
"disable_rss_feed"
]
disable_atom_feed
=
self
.
datastore
.
blog_configuration
[
"disable_atom_feed"
]
entries
=
[]
if
(
not
disable_rss_feed
)
or
(
not
disable_atom_feed
):
entries
+=
self
.
get_feed_entries
()
from
venc2.threads.feed
import
FeedThread
if
not
disable_atom_feed
:
FeedThread
(
self
.
datastore
,
self
.
theme
,
self
.
patterns
,
self
.
forbidden
,
"atom"
).
do
(
entries
,
self
.
export_path
,
self
.
relative_origin
,
"│ "
,
'└'
if
disable_atom_feed
and
not
self
.
datastore
.
enable_jsonld
else
'├'
)
if
not
disable_rss_feed
:
FeedThread
(
self
.
datastore
,
self
.
theme
,
self
.
patterns
,
self
.
forbidden
,
"rss"
).
do
(
entries
,
self
.
export_path
,
self
.
relative_origin
,
"│ "
,
'└'
if
not
self
.
datastore
.
enable_jsonld
else
'├'
)
if
self
.
datastore
.
enable_jsonld
:
from
venc2.prompt
import
notify
from
venc2.l10n
import
messages
import
json
notify
(
self
.
indentation_level
+
' └─ '
+
messages
.
generating_jsonld_doc
)
dump
=
json
.
dumps
(
self
.
datastore
.
root_as_jsonld
)
f
=
open
(
"blog/root.jsonld"
,
'w'
)
f
.
write
(
dump
)
def
JSONLD
(
self
,
argv
):
if
self
.
current_page
==
0
:
if
self
.
current_page
==
0
and
self
.
datastore
.
enable_jsonld
:
return
'<script type="application/ld+json" src="root.jsonld"></script>'
return
''
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment