boop.py 16.2 KB
Newer Older
1
2
3
#!/bin/env python3

import os
4
import sys
5
import shutil
6
import distutils.dir_util
7
import locale
berumuron's avatar
berumuron committed
8
import yaml
9
10
import datetime
import uuid
11
import re
12
from enum import Enum
13

14
import markdown
15
import pytz
16

17
import boopsy
18
import boopfeed
19
20


21
locale.setlocale(locale.LC_ALL, "")
22

23

24
25
26
27
28
class Environment(Enum):
    DEVELOPMENT = 1
    PRODUCTION = 2


29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
class ProgramError(Exception):
    """Exception raised during the program execution.

    Examples:

    >>> try:
    ...     raise ProgramError("Oops")
    ... except ProgramError as e:
    ...     print(e.args[0])
    Oops
    """

    pass


44
class Article:
45
    def __init__(self, slug, article_content, configuration={}, serie=None):
46
47
        # We convert the file's content into HTML
        md = markdown.Markdown(extensions=["meta", "fenced_code", "attr_list"])
48
        content = md.convert(article_content)
49
50
51
52

        # We get the local variables from the Markdown file (metadata)
        # which will be accessible in the article template
        self.meta = {}
53
54
        self.meta.update(configuration)
        self.meta["ARTICLE_SLUG"] = slug
55
        self.meta["ARTICLE_TEMPLATE"] = "article"
56
57
58
        for key, values in md.Meta.items():
            article_key = f"ARTICLE_{key.upper()}"
            # The Markdown's meta extension extract all the values in arrays, even
59
60
61
62
63
            # if there is only one value.
            if len(values) > 1:
                self.meta[article_key] = values
            else:
                self.meta[article_key] = values[0]
64
        self.meta["ARTICLE_CONTENT"] = content
65
        self.meta["ARTICLE_SERIE"] = serie
66
67

        # And we initialize a template for the article
68
69
70
        article_template_filepath = os.path.join(
            "templates", f"{self.meta['ARTICLE_TEMPLATE']}.html"
        )
71
72
73
        self.template = boopsy.Template(article_template_filepath)

    def render(self):
74
75
76
77
78
79
80
81
        return self.template.render(self.meta)

    def slug(self):
        return self.meta["ARTICLE_SLUG"]

    def title(self):
        return self.meta.get("ARTICLE_TITLE", "An article")

82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
    def author(self):
        if "ARTICLE_AUTHOR" in self.meta:
            return self.meta["ARTICLE_AUTHOR"]
        elif "SITE_AUTHOR" in self.meta:
            return self.meta["SITE_AUTHOR"]
        else:
            return None

    def url(self):
        site_url = self.meta.get("SITE_URL", "")
        return f"{site_url}{self.slug()}.html"

    def uuid(self):
        return str(uuid.uuid5(uuid.NAMESPACE_URL, self.url()))

97
98
    def content(self):
        return self.meta["ARTICLE_CONTENT"]
99

100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
    def timezone(self):
        site_timezone = self.meta.get("SITE_TIMEZONE", "UTC")
        return pytz.timezone(site_timezone)

    def date(self):
        if "ARTICLE_DATE" not in self.meta:
            return datetime.datetime.now(tz=self.timezone())

        article_date = datetime.datetime.strptime(
            self.meta["ARTICLE_DATE"], "%Y-%m-%d %H:%M"
        )
        return article_date.astimezone(tz=self.timezone())

    def update(self):
        if "ARTICLE_UPDATE" not in self.meta:
            return None

        article_update = datetime.datetime.strptime(
            self.meta["ARTICLE_UPDATE"], "%Y-%m-%d %H:%M"
        )
        return article_update.astimezone(tz=self.timezone())

122
123
124
    def serie(self):
        return self.meta["ARTICLE_SERIE"]

125

126
127
128
129
130
131
132
class Page:
    def __init__(self, slug, page_content, configuration={}):
        # We initialize the page meta variables which will be accessible in the
        # template
        self.meta = {}
        self.meta.update(configuration)
        self.meta["PAGE_SLUG"] = slug
133
        self.meta["PAGE_TEMPLATE"] = "page"
134
135
136
137
        yaml_conf, page_content = extract_yaml_configuration(page_content)
        for key, value in yaml_conf.items():
            article_key = f"PAGE_{key.upper()}"
            self.meta[article_key] = value
138
139
140
        self.meta["PAGE_CONTENT"] = page_content

        # And we initialize a template for the page
141
142
143
        page_template_filepath = os.path.join(
            "templates", f"{self.meta['PAGE_TEMPLATE']}.html"
        )
144
        self.template = None
145
146
147
148
149
150
151
152
153
154
155
156
        if os.path.exists(page_template_filepath):
            self.template = boopsy.Template(page_template_filepath)

    def render(self):
        if self.template is None:
            return self.content()
        else:
            return self.template.render(self.meta)

    def slug(self):
        return self.meta["PAGE_SLUG"]

157
158
159
    def title(self):
        return self.meta.get("PAGE_TITLE", "A page")

160
161
162
163
    def url(self):
        site_url = self.meta.get("SITE_URL", "")
        return f"{site_url}{self.slug()}.html"

berumuron's avatar
berumuron committed
164
165
166
    def uuid(self):
        return str(uuid.uuid5(uuid.NAMESPACE_URL, self.url()))

167
168
169
170
    def content(self):
        return self.meta["PAGE_CONTENT"]


171
172
173
174
175
176
177
178
179
180
181
182
YAML_FRONT_MATTER_REGEX = re.compile("(---\n.*\n)(---\n)(.*)", re.M | re.S)


def extract_yaml_configuration(content):
    match = YAML_FRONT_MATTER_REGEX.match(content)
    if match:
        yaml_conf = yaml.load(match.group(1))
        return yaml_conf, match.group(3)
    else:
        return {}, content


183
184
185
186
187
188
189
190
191
192
193
def build_article_from_filepath(filepath, configuration, serie=None):
    """Read a file and initialize an Article from it.
    """
    filepath_no_ext, _ = os.path.splitext(filepath)
    slug = os.path.basename(filepath_no_ext)
    with open(filepath, "r") as article_file:
        return Article(
            slug, article_file.read(), configuration=configuration, serie=serie
        )


194
195
196
197
198
199
200
201
202
def list_dirs(path):
    """List all the directories within a path.
    """
    for filename in os.listdir(path):
        full_path = os.path.join(path, filename)
        if os.path.isdir(full_path):
            yield filename


203
204
205
206
207
208
209
210
211
212
213
def match_extensions(filename, extensions):
    """Return True if filename extension is part of extensions array.
    """
    if len(extensions) > 0:
        _, ext = os.path.splitext(filename)
        if ext[1:] not in extensions:
            return False
    return True


def list_files(path, only=[]):
214
215
216
217
218
    """List all the files within a path.
    """
    for filename in os.listdir(path):
        full_path = os.path.join(path, filename)

219
220
221
222
223
224
225
226
        # We don't want to yield directories
        if not os.path.isfile(full_path):
            continue

        # If file extension is not part of "only" array, we don't yield it. If
        # only is empty, it means we accept all extensions.
        if not match_extensions(full_path, only):
            continue
227

228
229
230
231
        yield filename


def dir_tree(path, only=[]):
232
233
234
235
236
    """List all the tree directory under the given path.
    """
    for filename in os.listdir(path):
        full_path = os.path.join(path, filename)
        if os.path.isdir(full_path):
237
            for sub_filename in dir_tree(full_path, only=only):
238
                yield os.path.join(filename, sub_filename)
239
        elif match_extensions(filename, only):
240
241
242
            yield filename


berumuron's avatar
berumuron committed
243
244
245
246
247
248
249
def mkdirs_for_file(filepath):
    """Create parent directories for a given file.
    """
    path = os.path.dirname(filepath)
    os.makedirs(path, exist_ok=True)


250
251
252
def init_output_folder(output_path):
    """Create the output directory, empty it if it contains files.
    """
253
    if os.path.isdir(output_path):
254
        # it should be a directory...
255
        shutil.rmtree(output_path)
256
257
258
    elif os.path.exists(output_path):
        # ... but it also can be a simple file if user created it manually!
        os.remove(output_path)
259
    os.mkdir(output_path)
260

261
262

def load_configuration(configuration_path, output_path):
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
    configuration = {
        "SITE_URL": "filesystem",
        "SITE_TITLE": "A website",
        "SITE_TIMEZONE": "UTC",
    }

    if os.path.exists(configuration_path):
        with open(configuration_path) as conf_file:
            conf_data = yaml.load(conf_file)
        for key, value in conf_data.items():
            if key == "development":
                continue
            site_key = f"SITE_{key.upper()}"
            configuration[site_key] = value

        if environment == Environment.DEVELOPMENT and "development" in conf_data:
            for key, value in conf_data["development"].items():
                site_key = f"SITE_{key.upper()}"
                configuration[site_key] = value

    configuration["SITE_UUID"] = str(
        uuid.uuid5(uuid.NAMESPACE_URL, configuration["SITE_URL"])
    )
    if "SITE_AUTHOR" not in configuration:
        configuration["SITE_AUTHOR"] = f"{configuration['SITE_TITLE']}'s author"
    if configuration["SITE_URL"] == "filesystem":
        configuration["SITE_URL"] = f"file://{os.path.abspath(output_path)}/"

291
292
293
294
295
296
297
298
299
300
301
302
    return configuration


def main(environment):
    # Make sure ./site folder is empty
    output_path = os.path.join(os.curdir, "site")
    init_output_folder(output_path)

    # Load the configuration (it can be overidden by a configuration.yml file)
    configuration_path = os.path.join(os.curdir, "configuration.yml")
    configuration = load_configuration(configuration_path, output_path)

303
304
305
    # STEP 1: we load all the Articles and Pages
    pages = []
    articles = []
berumuron's avatar
berumuron committed
306
    articles_by_series = {}
307
308

    # Check that index file exists
309
310
311
    index_filepath = os.path.join(os.curdir, "index.html")
    if not os.path.exists(index_filepath):
        raise ProgramError("./index.html file does not exist")
312
313
314
315
316
317

    with open(index_filepath, "r") as index_file:
        page = Page("index", index_file.read(), configuration=configuration)
        pages.append(page)

    # Check that pages directory exists and load Pages from it
berumuron's avatar
berumuron committed
318
319
    pages_path = os.path.join(os.curdir, "pages")
    if os.path.isdir(pages_path):
320
        for filename in dir_tree(pages_path, only=["html"]):
321
            # We read the page file and initialize a Page from it
322
            slug, _ = os.path.splitext(filename)
berumuron's avatar
berumuron committed
323
            page_filepath = os.path.join(pages_path, filename)
324
325
326
327
            with open(page_filepath, "r") as page_file:
                page = Page(slug, page_file.read(), configuration=configuration)
                pages.append(page)

328
329
330
    # Check that articles directory exists and load Articles from it
    articles_path = os.path.join(os.curdir, "articles")
    if os.path.isdir(articles_path):
331
332
333
        # We iterate on the directories of articles dir. If they contain
        # "serie.html" file, it means they contain a serie. Otherwise, we list
        # articles as usual.
334
335
336
337
        dirs_with_articles = list(list_dirs(articles_path)) + ["."]
        for dirname in dirs_with_articles:
            dirpath = os.path.realpath(os.path.join(articles_path, dirname))

338
339
            serie_filepath = os.path.join(dirpath, "serie.html")
            serie = None
340
            if dirname != "." and os.path.exists(serie_filepath):
341
342
343
                # We have a serie Page! Each article of the directory must be
                # considered part of the serie.
                slug = f"serie/{dirname}"
berumuron's avatar
berumuron committed
344
345
346
                with open(serie_filepath, "r") as serie_file:
                    serie = Page(slug, serie_file.read(), configuration=configuration)
                    pages.append(serie)
347
                    articles_by_series[serie] = []
berumuron's avatar
berumuron committed
348

349
            for filename in list_files(dirpath, only=["md"]):
350
                article_filepath = os.path.join(dirpath, filename)
351
352
353
354
355
356
                article = build_article_from_filepath(
                    article_filepath, configuration, serie
                )
                articles.append(article)
                if serie:
                    articles_by_series[serie].append(article)
357

358
    # STEP 2: we write all the pages and articles
359
360
361
362
363
364
365
366
367
    for page in pages:
        output_filepath = os.path.join(output_path, f"{page.slug()}.html")

        # Create dir structure for the file
        mkdirs_for_file(output_filepath)

        # And write the file
        with open(output_filepath, "w") as output_file:
            output_file.write(page.render())
368
        print(f"Written page: {page.url()}")
369

370
371
    # Sort the articles by publication date
    articles.sort(key=lambda article: article.date(), reverse=True)
berumuron's avatar
berumuron committed
372
373
    for serie_articles in articles_by_series.values():
        serie_articles.sort(key=lambda article: article.date(), reverse=True)
374

375
376
    # And write them in the ./site folder
    for article in articles:
377
        output_filepath = os.path.join(output_path, f"{article.slug()}.html")
378
        with open(output_filepath, "w") as output_file:
379
            output_file.write(article.render())
380
        print(f"Written article: {article.url()}")
381

382
383
384
385
386
387
388
389
390
391
    # Write the page with the list of articles if blog template exists
    blog_template_filepath = os.path.join("templates", "blog.html")
    if os.path.exists(blog_template_filepath):
        blog_template = boopsy.Template(blog_template_filepath)
        context = {}
        context.update(configuration)
        context["ARTICLES"] = articles
        blog_output_filepath = os.path.join(output_path, f"blog.html")
        with open(blog_output_filepath, "w") as output_file:
            output_file.write(blog_template.render(context))
392
        print(f"Written page: {configuration['SITE_URL']}blog.html")
393

berumuron's avatar
berumuron committed
394
    # STEP 3: we write also the Atom feeds (the main one + the series').
395
396
397
398
399
400
401
402
    # Just create some variables so it's easier to manipulate after
    site_url = configuration["SITE_URL"]
    site_uuid = configuration["SITE_UUID"]
    site_title = configuration["SITE_TITLE"]
    site_timezone = configuration["SITE_TIMEZONE"]
    timezone = pytz.timezone(site_timezone)
    current_datetime = datetime.datetime.now(tz=timezone)

berumuron's avatar
berumuron committed
403
404
405
406
    feed_dirpath = os.path.join(output_path, "feeds")
    os.mkdir(feed_dirpath)

    # Build the main Atom feed
407
408
409
410
411
412
413
414
415
    feed = boopfeed.Atom()
    feed.add_header(
        {
            "title": site_title,
            "self_link": f"{site_url}feeds/all.atom.xml",
            "site_link": site_url,
            "id": f"urn:uuid:{site_uuid}",
            "updated": current_datetime,
        }
416
417
418
419
    )

    for article in articles:
        # and complete it for each article
420
421
422
423
424
425
426
427
        entry = {
            "title": article.title(),
            "id": f"urn:uuid:{article.uuid()}",
            "article_link": article.url(),
            "published": article.date(),
            "content": article.content(),
        }

428
429
        article_author = article.author()
        if article_author:
430
            entry["author"] = article_author
431
432
        article_update = article.update()
        if article_update:
433
434
435
            entry["updated"] = article_update

        feed.add_entry(entry)
436
437
438

    # And write it on the filesystem
    feed_filepath = os.path.join(feed_dirpath, "all.atom.xml")
439
    feed.write(feed_filepath)
440
441
    print(f"Written feed: {site_url}feeds/all.atom.xml")

berumuron's avatar
berumuron committed
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
    # Now, build the feeds for series (same process)
    for serie, serie_articles in articles_by_series.items():
        feed_filename = f"{os.path.basename(serie.slug())}.atom.xml"
        feed_filepath = os.path.join(feed_dirpath, feed_filename)

        feed = boopfeed.Atom()
        feed.add_header(
            {
                "title": f"{serie.title()} - {site_title}",
                "self_link": f"{site_url}feeds/{feed_filename}",
                "site_link": serie.url(),
                "id": f"urn:uuid:{serie.uuid()}",
                "updated": current_datetime,
            }
        )

        for article in serie_articles:
            # and complete it for each article
            entry = {
                "title": article.title(),
                "id": f"urn:uuid:{article.uuid()}",
                "article_link": article.url(),
                "published": article.date(),
                "content": article.content(),
            }

            article_author = article.author()
            if article_author:
                entry["author"] = article_author
            article_update = article.update()
            if article_update:
                entry["updated"] = article_update

            feed.add_entry(entry)

        # And write it on the filesystem
        feed.write(feed_filepath)
        print(f"Written feed: {site_url}feeds/{feed_filename}")

481
482
483
484
485
    # STEP 4: and we finish by copying the static folder.
    static_dirpath = os.path.join(os.curdir, "static")
    if os.path.isdir(static_dirpath):
        distutils.dir_util.copy_tree(static_dirpath, output_path)
        print(f"Static files copied")
486

487
488

if __name__ == "__main__":
489
490
491
492
    environment = Environment.PRODUCTION
    if len(sys.argv) == 2 and sys.argv[1] == "--development":
        environment = Environment.DEVELOPMENT

493
    try:
494
        main(environment)
495
496
497
        print("Boop!")
    except ProgramError as e:
        print(e.args[0], file=sys.stderr)