Commit 212eb0fc authored by setop's avatar setop 🚀
Browse files

fix handle void tags + minor typos

parent 43bcc487
......@@ -64,7 +64,7 @@ git clone this repo then cd inside.
Create ".env" file with variables seen in configuration part as `export KEY="VALUE"` lines.
To scrap, build feed and upload it, run : `./script/run_local.sh`
To scrap, build feed and upload it, run : `./scripts/runscrap.sh <feed_id> <feed_url>`
This could be put in a crontab.
......@@ -80,13 +80,14 @@ git clone this repo then cd inside.
`docker build -t scraplap .`
`docker save scraplap:latest | gzip -9 > scraplap.tgz`
### deploy
move the image to target environment
### run
`docker save scraplap:latest | pv | ssh <prod_host> docker load`
`gzip -cd scraplap.tgz | docker load`
### run
`useradd -b /var scrapy`
......@@ -96,7 +97,6 @@ To scrap, build feed and upload it, run :
```
docker run --rm -u $(id -u scrapy):$(id -g scrapy) \
--env-file .env \
--volume /var/scrapy:/var/scrapy \
--volume /var/scrapy:/scraplap/.scrapy \
scraplap
......@@ -104,8 +104,6 @@ docker run --rm -u $(id -u scrapy):$(id -g scrapy) \
Feed file and logs are produced on the host in `/var/scrapy`.
This could be put in a crontab.
# TODO
......
......@@ -20,7 +20,6 @@ class DBFeedExporter(BaseItemExporter):
conn = settings.db
item["pubdate"] = int(mktime(item["pubdate"]))
item["tags"] = ",".join(item["tags"])
print(*build_insert("entry",item))
conn.execute(*build_insert("entry",item))
conn.commit()
......
......@@ -37,7 +37,7 @@ db = None
# Enable and configure HTTP caching (disabled by default)
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
HTTPCACHE_ENABLED = True
HTTPCACHE_ENABLED = False # use cache in dev when debug, in prod it is not needed
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = 'httpcache'
HTTPCACHE_POLICY = 'lobsters.spiders.feed.AllButStartURLPolicy'
......
......@@ -42,8 +42,9 @@ def format_entry(xg: XMLGenerator, feed_name, i, t, d, l, tg, ct, c, n):
else:
xg.characters(c)
xg.endElement("content")
for tag in tg.split(","):
build_leaf_tag(xg, "category", {}, tag)
if tg is not None:
for tag in tg.split(","):
build_leaf_tag(xg, "category", {}, tag)
xg.endElement("entry")
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment