Commit 5aed4e36 authored by Stephane Bortzmeyer's avatar Stephane Bortzmeyer
Browse files

Stores the registered domain. Closes #4

parent e6e6588d
......@@ -17,9 +17,11 @@ python3 setup.py install --user
```
(On a Debian machine, the prerequitises are packages python3-pip,
python3-psycopg2, python3-openssl and the Gemini library [agunua](https://framagit.org/bortzmeyer/agunua). On other systems, pip3 will
python3-psycopg2, python3-openssl, the Gemini library
[agunua](https://framagit.org/bortzmeyer/agunua), and two unpackaged
PyPi sofwtare, py-scfg and public_suffix_list. On other systems, pip3 will
install the dependencies. You can also install them with `pip3 install
pyopenssl psycopg2 agunua py-scfg`.
pyopenssl psycopg2 agunua py-scfg public_suffix_list`.
Usage requires a [PostgreSQL](https://www.postgresql.org/) database,
to store the URLs and the result of crawling. Once you've created the
......
......@@ -5,6 +5,7 @@ CREATE TABLE Tlds (id SERIAL UNIQUE NOT NULL,
CREATE TABLE Capsules (id SERIAL UNIQUE NOT NULL,
name TEXT UNIQUE NOT NULL, -- If the port is not the standard one, the name will include the port.
tld INTEGER REFERENCES Tlds(id) NOT NULL,
registereddomain INTEGER REFERENCES RegisteredDomains(id) NOT NULL,
port INTEGER,
validcert BOOLEAN,
certissuer TEXT,
......@@ -16,10 +17,17 @@ CREATE TABLE Capsules (id SERIAL UNIQUE NOT NULL,
rawrobotstxt TEXT,
lasttest TIMESTAMP,
lastsuccessfulconnect TIMESTAMP,
lastrobotstxtretrieval TIMESTAMP,
lastrobotstxtretrieval TIMESTAMP, -- May be two columns
-- instead, one for attempts and one for success (if we get
-- a network success and a 51, we can erase robotstxt, not
-- if we had a network timeout.
denylisted TIMESTAMP,
added TIMESTAMP NOT NULL DEFAULT now());
CREATE TABLE RegisteredDomains (id SERIAL UNIQUE NOT NULL,
name TEXT UNIQUE NOT NULL,
tld INTEGER REFERENCES Tlds(id) NOT NULL);
CREATE TABLE Urls (id SERIAL UNIQUE NOT NULL,
url TEXT UNIQUE NOT NULL,
capsule INTEGER REFERENCES Capsules(id) NOT NULL,
......
......@@ -351,10 +351,11 @@ num_tlds = int(db.curs.fetchone()[0])
print("There are %i TLDs in the capsule's names." % (num_tlds, ), file=outfile)
# TODO Groupe capsules by SLD? Otherwise, *.flounder.online dominates
db.curs.execute("SELECT Tlds.name, count(Capsules.id) AS count FROM Tlds,Capsules WHERE Capsules.tld = Tlds.id GROUP by Tlds.name ORDER BY count DESC LIMIT %s;", (config.stats["max-tlds"], ))
print("""
### Most common TLDs (by number of capsules)
### Most common TLDs
#### By number of capsules)
(There's a strong bias towards TLDs which have hosting services such as flounder.online, which has many capsules in subdomains.)
......@@ -362,6 +363,15 @@ print("""
for capsule in db.curs.fetchall():
print("* %s: %i capsules" % (capsule[0], capsule[1]), file=outfile)
db.curs.execute("SELECT Tlds.name, count(RegisteredDomains.id) AS count FROM Tlds,RegisteredDomains WHERE RegisteredDomains.tld = Tlds.id GROUP by Tlds.name ORDER BY count DESC LIMIT %s;", (config.stats["max-tlds"], ))
print("""
#### By number of registered domains)
""", file=outfile)
for domain in db.curs.fetchall():
print("* %s: %i domains" % (domain[0], domain[1]), file=outfile)
# TODO and TLDs by number of links?
print("""
......
......@@ -9,6 +9,9 @@ import urllib.parse
import sys
import logging
# https://pypi.org/project/public-suffix-list/
import public_suffix_list
# https://framagit.org/bortzmeyer/agunua
import Agunua
import Agunua.urltinkering
......@@ -138,6 +141,7 @@ It does not use robots.txt or care about it.
return (True, False, None, None, url)
labels = host.split(".")
tld = labels[len(labels)-1]
reg_domain = public_suffix_list.registered_domain_name(host)
# Check there is a URL for the "home page"
if components.path != "" and components.path != "/":
test_url = "gemini://%s/" % (capsule, )
......@@ -164,10 +168,17 @@ It does not use robots.txt or care about it.
self.curs.execute("INSERT INTO Tlds (name) VALUES (%s)", (tld, ))
self.curs.execute("SELECT id FROM Tlds WHERE name = %s", (tld, ))
tld_ref = self.curs.fetchone()
self.curs.execute("SELECT id FROM RegisteredDomains WHERE name = %s", (reg_domain, ))
reg_domain_ref = self.curs.fetchone()
if reg_domain_ref is None:
self.curs.execute("INSERT INTO RegisteredDomains (name, tld) VALUES (%s, %s)", (reg_domain, tld_ref))
self.curs.execute("SELECT id FROM RegisteredDomains WHERE name = %s", (reg_domain, ))
reg_domain_ref = self.curs.fetchone()
self.curs.execute("SELECT id FROM Capsules WHERE name = %s", (capsule, ))
capsule_ref = self.curs.fetchone()
if capsule_ref is None:
self.curs.execute("INSERT INTO Capsules (name, tld, port) VALUES (%s, %s, %s)", (capsule, tld_ref, port))
self.curs.execute("INSERT INTO Capsules (name, tld, registereddomain, port) VALUES (%s, %s, %s, %s)",
(capsule, tld_ref, reg_domain_ref, port))
self.curs.execute("SELECT id FROM Capsules WHERE name = %s", (capsule, ))
capsule_ref = self.curs.fetchone()
self.curs.execute("INSERT INTO Urls (url, capsule) VALUES (%s, %s)", (url, capsule_ref))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment