Commit 181ee954 authored by thuban's avatar thuban

major update and cleanup + search speed optimization

parent 2b74767c
This diff is collapsed.
......@@ -89,10 +89,10 @@ def htmlget(url, params=None):
p = urllib.parse.urlencode(params)
if url.endswith('?'):
url = '{}{}'.format(url,p)
response = opener.open(url)
response = opener.open(url, timeout=10)
else:
p = p.encode('ascii')
response = opener.open(url, p)
response = opener.open(url, p, timeout=10)
else:
response = opener.open(url)
data = response.read()
......
......@@ -44,184 +44,6 @@ def do_req_get(url, ret, keys=None):
return retval
class OMDBbAPI:
"""Search on The Open Movie Database"""
URL = "https://www.omdbapi.com/"
def __init__(self):
pass
@classmethod
def search_by_title(cls, title, year="", plot="full"):
keys = {'t': title, 'r': 'json', 'plot': plot, 'y': year}
return do_req_get(cls.URL, "json", keys)
@classmethod
def search_by_id(cls, id, plot="full"):
keys = {'i': id, 'r': 'json', 'plot': plot}
return do_req_get(cls.URL, "json", keys)
@classmethod
def search_any(cls, sch):
keys = {'s': sch, 'r': 'json'}
return do_req_get(cls.URL, "json", keys)
def omdb_sch(sch, serie=False):
"""
Search on omdb for a movie or a serie. Default search is movie
Args:
sch (str): The title of the search
serie (boolean): If True, search series else search movies
Return:
List of results:
result = {'poster' : posterurl, 'title' : title, 'year' : year }
"""
results = []
title, year, posterurl = "", "", ""
search_omdb = OMDBbAPI.search_any(sch)
if not 'Response' in search_omdb.keys():
return {}
if search_omdb['Response']:
if 'Search' in search_omdb:
for res in search_omdb['Search']:
if serie:
if res['Type'] == "series":
title = res['Title']
year = res['Year']
posterurl = res['Poster']
if posterurl == 'N/A':
posterurl = "/static/unknown.png"
results.append({'poster': posterurl, 'title': title, 'year': year})
else:
if res['Type'] == "movie":
title = res['Title']
year = res['Year']
posterurl = res['Poster']
if posterurl == 'N/A':
posterurl = "/static/unknown.png"
results.append({'poster': posterurl, 'title': title, 'year': year})
return results
def get_episode_list(title, year):
"""
Args:
title:
year:
Return:
List of available episodes of the serie 'title':
episodelist = [{ep1}, {ep2}]
"""
url = "http://www.omdbapi.com/"
season = 1
eplist = []
# first try
keys = {'t': title, 'Season': season, 'y': year}
json = do_req_get(url, "json", keys)
while json['Response'] == 'True':
new = {'Season': json['Season'], 'Episodes': json['Episodes']}
eplist.append(new)
season += 1
keys = {'t': title, 'Season': season, 'y': year}
json = do_req_get(url, "json", keys)
# clean the list of unreleasead or unknown release date
# for i in eplist:
# if i['Released'] == "N/A" or \
# re.search(r'Episode #[0-9]?\.[0-9]?', i['Title']) != None :
# eplist.remove(i)
return eplist
class BestIMDB:
URL_TVSHOWS = "http://www.imdb.com/chart/tvmeter?ref_=nv_tvv_mptv_4"
URL_DVD = "http://www.imdb.com/sections/dvd/?ref_=nv_tvv_dvd_6"
def __init__(self):
pass
@classmethod
def get_best_dvd(cls):
s = do_req_get(cls.URL_DVD, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
results = soup.find_all("div", class_="list_item")
elements = []
for result in results:
id = result.find("div", class_="info").b.a["href"]
id = id[len("/title/"):]
id = id[:id.find("/")]
search_omdb = OMDBbAPI.search_by_id(id, plot="short")
title = search_omdb["Title"]
year = search_omdb["Year"]
img = search_omdb["Poster"]
elements.append({"title": title, "year": year, "img": img})
return elements
@classmethod
def get_best_tvshows(cls, limit=25):
s = do_req_get(cls.URL_TVSHOWS, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
results = soup.find("tbody", class_="lister-list")
results = results.find_all("tr")
elements = []
i = 0
for result in results:
id = result.find("td", class_="titleColumn").a["href"]
id = id[len("/title/"):]
id = id[:id.find("/")]
search_omdb = OMDBbAPI.search_by_id(id, plot="short")
title = search_omdb["Title"]
year = search_omdb["Year"]
img = search_omdb["Poster"]
search_carac = "–"
if year.endswith(search_carac): # Warning: the caracter here is not a regular minus
year += _("Today")
elements.append({"title": title, "year": year, "img": img})
# If we don't do that, we retrieve the first 100 results! It's too long.
if i == limit:
break
i += 1
return elements
@classmethod
def url_best_dvd(cls):
return cls.URL_DVD
@classmethod
def url_best_tvshows(cls):
return cls.URL_TVSHOWS
class TorrentSearch:
def __init__(self, searchengine):
......@@ -246,8 +68,7 @@ class TorrentSearch:
elif self.searchengine == 'isohunt':
s = [res for res in self._isohunt(sch)]
elif self.searchengine == 'torrent9':
for res in self._torrent9(sch):
yield res
s = [res for res in self._torrent9(sch)]
elif self.searchengine == 'alphareign':
s = [res for res in self._alphareign(sch, page)]
elif self.searchengine == 'btdb':
......@@ -361,9 +182,8 @@ class TorrentSearch:
size : size
seed : number of seeds
"""
url = 'https://thepiratebay.se/s/'
keys = {'q': sch}
s = do_req_get(url, "text", keys)
url = 'https://thepiratebay.org/search/{}///0'.format(sch)
s = do_req_get(url, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
name = str()
......@@ -382,7 +202,8 @@ class TorrentSearch:
for magnet_result in alinks:
href = magnet_result['href']
if href.startswith('magnet:'):
magnet = href.split('&tr=')[0] # to remove trackers
magnet = href
#magnet = href.split('&tr=')[0] # to remove trackers
if not size:
size_result = i.find('font', class_='detDesc')
......@@ -525,11 +346,12 @@ class TorrentSearch:
name = ' '.join(name)
else:
if not size:
size = d.string
size = d.text
elif not seeds :
seeds = d.string
seeds = d.text
elif not leechs :
leechs = d.string
leechs = d.text
if name and magnet and size and seeds and leechs :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
......@@ -588,7 +410,6 @@ class TorrentSearch:
magnet = link['href']
if name and magnet and size and seeds and leechs :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
name = str()
magnet = str()
......@@ -705,6 +526,7 @@ class TorrentSearch:
size = str()
seeds = '?'
leechs = '?'
tmplist = []
res = {}
for li in soup.find_all('li'):
......@@ -712,12 +534,7 @@ class TorrentSearch:
param = li.find('p', class_="param")
if title :
name = title.a['title']
href = title.a['href']
# look for magnet link
u = '{}{}'.format(baseurl,href)
mgthtml = do_req_get(u, "text")
mgtsoup = bs4.BeautifulSoup(mgthtml, "html.parser")
magnet = mgtsoup.find('textarea', class_="magnet-link").text
magnet = title.a['href']
if param:
size = param.text.splitlines()[3]
......@@ -725,11 +542,24 @@ class TorrentSearch:
if name and magnet and size :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
tmplist.append(res)
name = str()
magnet = str()
size = str()
yield res
with ThreadPoolExecutor(max_workers=5) as executor:
future_mgt = { executor.submit(do_req_get, "{}{}".format(baseurl,res['magnet']), "text") : res for res in tmplist }
for f in as_completed(future_mgt):
item = future_mgt[f]
ms = f.result()
mgtsoup = bs4.BeautifulSoup(ms, "html.parser")
magnet = mgtsoup.find('textarea', class_="magnet-link").text
item['magnet'] = magnet
yield item
def isohunt_parse_results(z):
"""
......@@ -759,9 +589,9 @@ def isohunt_parse_results(z):
if __name__ == '__main__':
with TorrentSearch('torrent9') as ts:
for result in ts.search('vikings vostfr', 0):
print(result)
with TorrentSearch('p2psearch') as ts:
s = ts.search('vikings vostfr', 0)
print(s)
......
<!doctype html>
<html>
%include('head.tpl')
<body>
<header>
<h1>{{title}}</h1>
<nav>
%include('backtomain.tpl')
<span class="float-right">
{{tpl_provided}} :
<span class="big"><a href="{{url}}">IMDB</a></span>
</span>
</nav>
</header>
<main> {{ !htmlstuff }} </main>
% include('footer.tpl')
% include('scripts.tpl')
<script>
$(function () {
$('[data-toggle=tooltip]').tooltip();
});
</script>
</body>
</html>
\ No newline at end of file
......@@ -20,19 +20,6 @@
<div id="scroll">
{{ !sch_res }}
</div>
%if (moviessch != '' or seriessch != '') and nocatalog != '1':
<h2>{{tpl_see_also}}</h2>
%if moviessch != '':
<h3>{{tpl_movies}}</h3>
{{ !moviessch }}
%end
%if seriessch != '':
<h3>{{tpl_series}}</h3>
{{ !seriessch }}
%end
%end
</main>
% include('footer.tpl')
......
......@@ -22,13 +22,6 @@
</div>
%end
<aside class="bottom">
<p>
<a href="/_bestvideos/0">{{tpl_best_dvd}}</a>
-
<a href="/_bestvideos/1">{{tpl_popular_series}}</a>
</p>
</aside>
</main>
%if islastv == False:
<a class="toupgradealert" href="http://3hg.toile-libre.org/download.html" target="_blank">{{tpl_not_last_version}}</a>
......
<!doctype html>
<html>
%include('head.tpl')
<body>
<header>
<h1>{{title}}</h1>
<h4 class="text-center" style="margin-bottom: 15px">
<span>{{ year }}</span>
<span style="margin-left: 45px">Rating: {{ rating }} / 10</span>
</h4>
<nav>
%include('backtomain.tpl')
</nav>
</header>
<main class="text-left">
%if html != "" :
{{ html }}
%end
%if isserie == "0" :
<div class="row">
<div class="col-5 text-center">
<img src="{{ poster }}" class="plotposter">
</div>
<div class="col-7">
<p>{{tpl_director}} {{director}}</p>
<p>{{tpl_writer}} {{writer}}</p>
<p>{{tpl_actors}} {{actors}}</p>
<div class="plot">
<p class="text-justify">{{plot}}</p>
<div class="text-right">
<a href="https://translate.google.com/#en/{{lang}}/{{plot}}" target="_blank">
{{tpl_google_translate}}
</a>
</div>
</div>
<form action="/search" method="get">
<input name="keywords" type="hidden" id="keywords" value="{{title_simplify}}">
<input name="nocatalog" type="hidden" value="1">
<p>{{tpl_extrawords}}
<input name="extrawords" type="text" placeholder="vostfr">
</p>
<p>{{tpl_searchon}}
{{!search_engines}}
</p>
<div class="text-center">
<input value="{{tpl_watchnow}}" type="submit" class="big">
</div>
</form>
</div>
</div>
%else:
<div class="row">
<div class="col-5">
<div>
<img src="{{ poster }}" class="plotposter">
<p>{{tpl_director}} {{director}}</p>
<p>{{tpl_writer}} {{writer}}</p>
<p>{{tpl_actors}} {{actors}}</p>
</div>
<div class="plot">
<p class="text-justify">{{plot}}</p>
<div class="text-right">
<a href="https://translate.google.com/#en/{{lang}}/{{plot}}" target="_blank">
{{tpl_google_translate}}
</a>
</div>
</div>
</div>
<div class="col-7">
<form action="/search" method="get" id="series_form">
<input name="keywords" type="hidden" id="keywords" value="">
<input name="nocatalog" type="hidden" value="1">
<input name="isserie" type="hidden" value="1">
<ul>
<li>{{tpl_extrawords}}
<input name="extrawords" type="text" placeholder="vostfr">
</li>
<li>{{tpl_searchon}}
{{!search_engines}}
</li>
</ul>
{{!list_ep_serie}}
</form>
</div>
</div>
%end
</main>
% include('footer.tpl')
% include('scripts.tpl')
</body>
</html>
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment