Commit 181ee954 authored by thuban's avatar thuban

major update and cleanup + search speed optimization

parent 2b74767c
......@@ -66,7 +66,7 @@ def search2html(keywords, extrawords=' ', searchengine='alphareign', page=0, iss
html += '<td class="text-center"><a href="{}">{}</a></td>\n'.format(link, result['leechs'])
# color depending on seeds number
color = 'white'
color = 'grey'
if result['seeds'] != '?':
seeds = int(result['seeds'])
if seeds <= 10:
......@@ -95,141 +95,6 @@ def search2html(keywords, extrawords=' ', searchengine='alphareign', page=0, iss
return html
def sch2html(keywords, serie=False):
"""
Args:
keywords:
serie:
Return:
Return a research on omdb in html code
"""
results = omdb_sch(keywords, serie=serie)
if len(results) == 0:
return ''
else:
html = '<ul class="ca-menu">'
for r in results:
isserie = 1 if serie else 0
html += '<li><a href="/whatsthat/{1}/{2}/{3}"><img src="{0}"><p>{1} <br> {2}</p></a></li>'.format(
r['poster'], r['title'], r['year'], isserie)
html += '</ul>'
return html
def getseriestuff(keywords, year, user=''):
serie_stuff = ''
r = OMDBbAPI.search_by_title(keywords, year)
if r['Response'] == 'True':
eplist = get_episode_list(keywords, year)
serie_stuff = '<div id="scroll">\n'
serie_stuff += '<table class="sortable">\n'
serie_stuff += '<thead>\n<tr>\n<th>{}</th>\n'.format(_('Season'))
serie_stuff += '<th>{}</th>\n'.format(_('Episode'))
serie_stuff += '<th>{}</th>\n'.format(_('Title'))
serie_stuff += '<th class="text-right">{}</th>\n'.format(_('Release date'))
serie_stuff += '<th>{}</th>\n</tr>\n</thead>\n'.format(_('Seen'))
serie_stuff += '<tbody>\n'
for season in eplist:
for e in season['Episodes']:
episode = 'S{0:02d}E{1:02d}'.format(int(season['Season']), int(e['Episode']))
search = '{0} {1}'.format(keywords, episode)
serie_stuff += '<tr>\n'
serie_stuff += '<td class="text-center">' \
'\t<a href="#" onClick="updateInput(\'{}\')">{:02d}</a>' \
'</td>\n'.format(search, int(season['Season']))
serie_stuff += '<td class="text-center">' \
'\t<a href="#" onClick="updateInput(\'{}\')">{:02d}</a>' \
'</td>\n'.format(search, int(e['Episode']))
serie_stuff += '<td>' \
'\t<a id="submit" href="#" onClick="updateInput(\'{}\')">{}</a>' \
'</td>\n'.format(search, e['Title'])
serie_stuff += '<td class="text-right">' \
'\t<a href="#" onClick="updateInput(\'{}\')">{}</a>' \
'</td>\n'.format(search, e['Released'])
with DontPanicDB() as db:
if db.is_user_in(user):
togglelink = "/_toggleep/{}/".format(search)
color, seen = ("green", _("Yes")) if db.is_title_in_wl(user, search) else ("red", _("No"))
serie_stuff += '<td class="text-center" style="background:{1};">' \
'<a href="{0}">{2}</a>' \
'</td>\n'.format(togglelink, color, seen)
else:
serie_stuff += '<td class="text-right">?</td>\n'
serie_stuff += '</tr>\n'
serie_stuff += '</tbody>\n'
serie_stuff += '</table>\n'
serie_stuff += '</div>\n'
serie_stuff += """
<script src="/static/perfect-scrollbar.min.js"></script>
<script>
window.onload = function() {
var container = document.getElementById("scroll");
Ps.initialize(container);
}
function updateInput(ish){
document.getElementById('keywords').value = ish;
document.getElementById('series_form').submit();
}
</script>"""
return serie_stuff
def omdb2template(keywords, year, isserie=0):
title = ''
runtime = ''
director = ''
writer = ''
actors = ''
poster = ''
plot = ''
rating = ''
html = ''
r = OMDBbAPI.search_by_title(keywords, year)
if 'Response' not in r.keys():
return title, runtime, director, writer, actors, poster, plot, rating, html
elif r['Response'] == 'True':
title = r['Title']
runtime = r['Runtime']
director = r['Director']
writer = r['Writer']
actors = r['Actors']
poster = r['Poster']
if poster == 'N/A':
poster = "/static/unknown.png"
plot = r['Plot']
rating = r['imdbRating']
else:
html = _("Do you mean ? ")
html += "<p>"
lop = []
k = keywords
while len(k.split(' ')) > 1:
k = k.rsplit(' ', 1)[0]
lop.append(k)
k = keywords
while len(k.split(' ')) > 1:
k = k.split(' ', 1)[-1]
lop.append(k)
for i in lop:
html += ' - '
html += '<a href="/whatsthat/{0}/{1}/{2}">{0}</a>'.format(i, year, isserie)
html += ' - '
html += "</p>"
return title, runtime, director, writer, actors, poster, plot, rating, html
def watchlist(user):
movies = ''
......@@ -351,26 +216,6 @@ class DontPanic:
if user == '':
user = request.get_cookie("username", secret=self.secret)
if user:
with DontPanicDB() as database:
recent_search = database.get_recent_search(user)
if len(recent_search) > 0:
for ele in recent_search:
title = ele["title"]
year = ele["year"]
isserie = ele["isserie"]
poster = ele["poster"]
posterhtml = '<img src="{}" title="{} {}" data-toggle="tooltip" ' \
'data-placement="bottom">'.format(poster, title, year)
posterhtml = '<a href="/whatsthat/{}/{}/{}">{}</a>'.format(title, year, isserie, posterhtml)
posterhtml += '<a class="delete" href="/_removeinrecent/{}/">✗</a>'.format(title)
posterhtml = '<div class="show-image">{}</div>'.format(posterhtml)
html += '<li>{}</li>'.format(posterhtml)
html = '<ul class="ca-menu">{}</ul>'.format(html)
tplvars = {'welcomemsg': pick_welcomemsg(),
'recents': html,
......@@ -489,25 +334,17 @@ class DontPanic:
if not isserie:
isserie = '0'
seriessch, moviessch = '', ''
logger.info('Searching on {}... '.format(searchengine))
# FIXME: Better handle when a search engine has modified its html code
try:
htmlresults = search2html(keywords, extrawords=extrawords, searchengine=searchengine, isserie=isserie)
except Exception as e:
print(e)
logger.info('Error while searching : {}... '.format(e))
htmlresults = '<div><br>{}</div>'.format(_("Error while searching. Try another engine"))
if nocatalog != "1":
logger.info('Searching {}... '.format("in series"))
seriessch = sch2html(keywords, serie=True)
logger.info('Searching {}... '.format("in movies"))
moviessch = sch2html(keywords, serie=False)
tplvars = {'sch_res': htmlresults,
'the_search': '{} {}'.format(keywords, extrawords).strip(),
'moviessch': moviessch,
'seriessch': seriessch,
'isserie': isserie,
'search_engines': create_select_search_engine(searchengine),
'nocatalog': nocatalog}
......@@ -530,48 +367,11 @@ class DontPanic:
tplvars = {
'sch_res': htmlresults,
'the_search': '{} {}'.format(keywords, extrawords).strip(),
'moviessch': '',
'seriessch': '',
'search_engines': create_select_search_engine(searchengine),
'isserie': isserie}
tplvars.update(tpl_msg)
return template('results', tplvars)
@self.app.route('/whatsthat/<title>/<year>/<isserie>')
def whatsthat(title, year, isserie):
title_simplify = simplifyname(title)
title, runtime, director, writer, actors, poster, plot, rating, html = omdb2template(
title_simplify, year, isserie)
whoami = request.get_cookie("username", secret=self.secret)
self._add_to_recent_search(whoami, title, year, poster, isserie)
if isserie:
whoami = request.get_cookie("username", secret=self.secret)
list_ep_serie = getseriestuff(title, year, user=whoami)
try:
lang = locale.setlocale(locale.LC_ALL, "")[0:2] # 'fr' or 'en'
except:
lang = 'en'
tplvars = {
'title': title,
'title_simplify': title_simplify,
'year': year,
'runtime': runtime,
'director': director,
'writer': writer,
'actors': actors,
'poster': poster,
'plot': plot,
'lang': lang,
'rating': rating,
'html': html,
'isserie': isserie,
'list_ep_serie': list_ep_serie,
'search_engines': create_select_search_engine(self.fav_sch_engine)}
tplvars.update(tpl_msg)
return template('what', tplvars)
@self.app.route("/stream/<name>/<magnet:path>/<isserie>")
def stream(name, magnet, isserie):
......@@ -875,36 +675,6 @@ class DontPanic:
database.remove_in_recent(whoami, title)
redirect(link)
@self.app.route('/_bestvideos/<i:int>')
def best_videos(i):
# Set source from IMDB
best = BestIMDB()
# See the best series
if i == 1:
url = best.url_best_tvshows()
title = "Popular series"
results = best.get_best_tvshows()
# See the best dvd's
else:
url = best.url_best_dvd()
title = "Best DVD's"
results = best.get_best_dvd()
html = str()
for result in results:
html += '<li><a href="/whatsthat/{1}/{2}/{3}"><img src="{0}" data-toggle="tooltip" ' \
'data-placement="bottom" title="{1} {2}"></a></li>'.format(result["img"], result["title"],
result["year"], i)
html = '<ul class="ca-menu">{}</ul>'.format(html)
tplvars = {'url': url, 'title': title, 'htmlstuff': html}
tplvars.update(tpl_msg)
return template('best', tplvars)
def read_config(self):
conf = load_config()
self.startperc = conf['startperc'] # percent to start a movie
......
......@@ -89,10 +89,10 @@ def htmlget(url, params=None):
p = urllib.parse.urlencode(params)
if url.endswith('?'):
url = '{}{}'.format(url,p)
response = opener.open(url)
response = opener.open(url, timeout=10)
else:
p = p.encode('ascii')
response = opener.open(url, p)
response = opener.open(url, p, timeout=10)
else:
response = opener.open(url)
data = response.read()
......
......@@ -44,184 +44,6 @@ def do_req_get(url, ret, keys=None):
return retval
class OMDBbAPI:
"""Search on The Open Movie Database"""
URL = "https://www.omdbapi.com/"
def __init__(self):
pass
@classmethod
def search_by_title(cls, title, year="", plot="full"):
keys = {'t': title, 'r': 'json', 'plot': plot, 'y': year}
return do_req_get(cls.URL, "json", keys)
@classmethod
def search_by_id(cls, id, plot="full"):
keys = {'i': id, 'r': 'json', 'plot': plot}
return do_req_get(cls.URL, "json", keys)
@classmethod
def search_any(cls, sch):
keys = {'s': sch, 'r': 'json'}
return do_req_get(cls.URL, "json", keys)
def omdb_sch(sch, serie=False):
"""
Search on omdb for a movie or a serie. Default search is movie
Args:
sch (str): The title of the search
serie (boolean): If True, search series else search movies
Return:
List of results:
result = {'poster' : posterurl, 'title' : title, 'year' : year }
"""
results = []
title, year, posterurl = "", "", ""
search_omdb = OMDBbAPI.search_any(sch)
if not 'Response' in search_omdb.keys():
return {}
if search_omdb['Response']:
if 'Search' in search_omdb:
for res in search_omdb['Search']:
if serie:
if res['Type'] == "series":
title = res['Title']
year = res['Year']
posterurl = res['Poster']
if posterurl == 'N/A':
posterurl = "/static/unknown.png"
results.append({'poster': posterurl, 'title': title, 'year': year})
else:
if res['Type'] == "movie":
title = res['Title']
year = res['Year']
posterurl = res['Poster']
if posterurl == 'N/A':
posterurl = "/static/unknown.png"
results.append({'poster': posterurl, 'title': title, 'year': year})
return results
def get_episode_list(title, year):
"""
Args:
title:
year:
Return:
List of available episodes of the serie 'title':
episodelist = [{ep1}, {ep2}]
"""
url = "http://www.omdbapi.com/"
season = 1
eplist = []
# first try
keys = {'t': title, 'Season': season, 'y': year}
json = do_req_get(url, "json", keys)
while json['Response'] == 'True':
new = {'Season': json['Season'], 'Episodes': json['Episodes']}
eplist.append(new)
season += 1
keys = {'t': title, 'Season': season, 'y': year}
json = do_req_get(url, "json", keys)
# clean the list of unreleasead or unknown release date
# for i in eplist:
# if i['Released'] == "N/A" or \
# re.search(r'Episode #[0-9]?\.[0-9]?', i['Title']) != None :
# eplist.remove(i)
return eplist
class BestIMDB:
URL_TVSHOWS = "http://www.imdb.com/chart/tvmeter?ref_=nv_tvv_mptv_4"
URL_DVD = "http://www.imdb.com/sections/dvd/?ref_=nv_tvv_dvd_6"
def __init__(self):
pass
@classmethod
def get_best_dvd(cls):
s = do_req_get(cls.URL_DVD, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
results = soup.find_all("div", class_="list_item")
elements = []
for result in results:
id = result.find("div", class_="info").b.a["href"]
id = id[len("/title/"):]
id = id[:id.find("/")]
search_omdb = OMDBbAPI.search_by_id(id, plot="short")
title = search_omdb["Title"]
year = search_omdb["Year"]
img = search_omdb["Poster"]
elements.append({"title": title, "year": year, "img": img})
return elements
@classmethod
def get_best_tvshows(cls, limit=25):
s = do_req_get(cls.URL_TVSHOWS, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
results = soup.find("tbody", class_="lister-list")
results = results.find_all("tr")
elements = []
i = 0
for result in results:
id = result.find("td", class_="titleColumn").a["href"]
id = id[len("/title/"):]
id = id[:id.find("/")]
search_omdb = OMDBbAPI.search_by_id(id, plot="short")
title = search_omdb["Title"]
year = search_omdb["Year"]
img = search_omdb["Poster"]
search_carac = "–"
if year.endswith(search_carac): # Warning: the caracter here is not a regular minus
year += _("Today")
elements.append({"title": title, "year": year, "img": img})
# If we don't do that, we retrieve the first 100 results! It's too long.
if i == limit:
break
i += 1
return elements
@classmethod
def url_best_dvd(cls):
return cls.URL_DVD
@classmethod
def url_best_tvshows(cls):
return cls.URL_TVSHOWS
class TorrentSearch:
def __init__(self, searchengine):
......@@ -246,8 +68,7 @@ class TorrentSearch:
elif self.searchengine == 'isohunt':
s = [res for res in self._isohunt(sch)]
elif self.searchengine == 'torrent9':
for res in self._torrent9(sch):
yield res
s = [res for res in self._torrent9(sch)]
elif self.searchengine == 'alphareign':
s = [res for res in self._alphareign(sch, page)]
elif self.searchengine == 'btdb':
......@@ -263,7 +84,7 @@ class TorrentSearch:
except Exception as e:
print(e)
s = []
return s
return s
def _search_all(self, sch):
s = []
......@@ -361,9 +182,8 @@ class TorrentSearch:
size : size
seed : number of seeds
"""
url = 'https://thepiratebay.se/s/'
keys = {'q': sch}
s = do_req_get(url, "text", keys)
url = 'https://thepiratebay.org/search/{}///0'.format(sch)
s = do_req_get(url, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
name = str()
......@@ -382,7 +202,8 @@ class TorrentSearch:
for magnet_result in alinks:
href = magnet_result['href']
if href.startswith('magnet:'):
magnet = href.split('&tr=')[0] # to remove trackers
magnet = href
#magnet = href.split('&tr=')[0] # to remove trackers
if not size:
size_result = i.find('font', class_='detDesc')
......@@ -525,11 +346,12 @@ class TorrentSearch:
name = ' '.join(name)
else:
if not size:
size = d.string
size = d.text
elif not seeds :
seeds = d.string
seeds = d.text
elif not leechs :
leechs = d.string
leechs = d.text
if name and magnet and size and seeds and leechs :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
......@@ -588,7 +410,6 @@ class TorrentSearch:
magnet = link['href']
if name and magnet and size and seeds and leechs :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
name = str()
magnet = str()
......@@ -705,6 +526,7 @@ class TorrentSearch:
size = str()
seeds = '?'
leechs = '?'
tmplist = []
res = {}
for li in soup.find_all('li'):
......@@ -712,24 +534,32 @@ class TorrentSearch:
param = li.find('p', class_="param")
if title :
name = title.a['title']
href = title.a['href']
# look for magnet link
u = '{}{}'.format(baseurl,href)
mgthtml = do_req_get(u, "text")
mgtsoup = bs4.BeautifulSoup(mgthtml, "html.parser")
magnet = mgtsoup.find('textarea', class_="magnet-link").text
magnet = title.a['href']
if param:
size = param.text.splitlines()[3]
size = size.strip().split(':')[1]
if name and magnet and size :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
tmplist.append(res)
name = str()
magnet = str()
size = str()
with ThreadPoolExecutor(max_workers=5) as executor:
future_mgt = { executor.submit(do_req_get, "{}{}".format(baseurl,res['magnet']), "text") : res for res in tmplist }
for f in as_completed(future_mgt):
item = future_mgt[f]
ms = f.result()
mgtsoup = bs4.BeautifulSoup(ms, "html.parser")
magnet = mgtsoup.find('textarea', class_="magnet-link").text
item['magnet'] = magnet
yield item
yield res
def isohunt_parse_results(z):
"""
......@@ -759,9 +589,9 @@ def isohunt_parse_results(z):
if __name__ == '__main__':
with TorrentSearch('torrent9') as ts:
for result in ts.search('vikings vostfr', 0):
print(result)
with TorrentSearch('p2psearch') as ts:
s = ts.search('vikings vostfr', 0)
print(s)
......
<!doctype html>
<html>
%include('head.tpl')
<body>
<header>
<h1>{{title}}</h1>
<nav>
%include('backtomain.tpl')
<span class="float-right">
{{tpl_provided}} :
<span class="big"><a href="{{url}}">IMDB</a></span>
</span>
</nav>
</header>
<main> {{ !htmlstuff }} </main>
% include('footer.tpl')
% include('scripts.tpl')
<script>
$(function () {
$('[data-toggle=tooltip]').tooltip();
});
</script>
</body>
</html>
\ No newline at end of file
......@@ -20,19 +20,6 @@
<div id="scroll">
{{ !sch_res }}
</div>
%if (moviessch != '' or seriessch != '') and nocatalog != '1':
<h2>{{tpl_see_also}}</h2>
%if moviessch != '':
<h3>{{tpl_movies}}</h3>
{{ !moviessch }}
%end
%if seriessch != '':
<h3>{{tpl_series}}</h3>
{{ !seriessch }}
%end
%end
</main>
% include('footer.tpl')
......
......@@ -22,13 +22,6 @@
</div>
%end
<aside class="bottom">
<p>
<a href="/_bestvideos/0">{{tpl_best_dvd}}</a>
-
<a href="/_bestvideos/1">{{tpl_popular_series}}</a>
</p>
</aside>
</main>
%if islastv == False:
<a class="toupgradealert" href="http://3hg.toile-libre.org/download.html" target="_blank">{{tpl_not_last_version}}</a>
......
<!doctype html>
<html>
%include('head.tpl')
<body>
<header>
<h1>{{title}}</h1>
<h4 class="text-center" style="margin-bottom: 15px">
<span>{{ year }}</span>
<span style="margin-left: 45px">Rating: {{ rating }} / 10</span>
</h4>
<nav>
%include('backtomain.tpl')
</nav>
</header>
<main class="text-left">
%if html != "" :
{{ html }}
%end
%if isserie == "0" :
<div class="row">
<div class="col-5 text-center">
<img src="{{ poster }}" class="plotposter">
</div>
<div class="col-7">
<p>{{tpl_director}} {{director}}</p>
<p>{{tpl_writer}} {{writer}}</p>
<p>{{tpl_actors}} {{actors}}</p>
<div class="plot">
<p class="text-justify">{{plot}}</p>
<div class="text-right">
<a href="https://translate.google.com/#en/{{lang}}/{{plot}}" target="_blank">
{{tpl_google_translate}}
</a>
</div>
</div>
<form action="/search" method="get">
<input name="keywords" type="hidden" id="keywords" value="{{title_simplify}}">
<input name="nocatalog" type="hidden" value="1">
<p>{{tpl_extrawords}}
<input name="extrawords" type="text" placeholder="vostfr">
</p>
<p>{{tpl_searchon}}
{{!search_engines}}
</p>
<div class="text-center">
<input value="{{tpl_watchnow}}" type="submit" class="big">
</div>
</form>
</div>
</div>
%else: