Commit 691be107 authored by prx's avatar prx

multithread. Debut

parent bf66ea67
......@@ -12,6 +12,7 @@ except:
from .i18nlib import _
from concurrent.futures import ThreadPoolExecutor
from concurrent.futures import as_completed
def do_req_get(url, ret, keys=None):
"""
......@@ -245,7 +246,8 @@ class TorrentSearch:
elif self.searchengine == 'isohunt':
s = [res for res in self._isohunt(sch)]
elif self.searchengine == 'torrent9':
s = [res for res in self._torrent9(sch)]
for res in self._torrent9(sch):
yield res
elif self.searchengine == 'alphareign':
s = [res for res in self._alphareign(sch, page)]
elif self.searchengine == 'btdb':
......@@ -261,28 +263,17 @@ class TorrentSearch:
except Exception as e:
print(e)
s = []
return s
return s
def _search_all(self, sch):
s = []
searchengine_list = [self._piratebay, self._kickasstorrents, self._isohunt, self._torrent9, self._alphareign, self._btdb, self._digbt, self._p2psearch]
if sys.version_info >= (3, 0):
with ThreadPoolExecutor(max_workers=5) as e:
for searchengine in searchengine_list:
try :
e.map(s.append, searchengine(sch))
except:pass
else:
s += [res for res in self._piratebay(sch)]
s += [res for res in self._kickasstorrents(sch)]
s += [res for res in self._isohunt(sch)]
s += [res for res in self._torrent9(sch)]
s += [res for res in self._alphareign(sch)]
s += [res for res in self._btdb(sch)]
s += [res for res in self._digbt(sch)]
s += [res for res in self._p2psearch(sch)]
with ThreadPoolExecutor(max_workers=5) as e:
for searchengine in searchengine_list:
try :
e.map(s.append, searchengine(sch))
except Exception as e:
print(e)
return s
def _btdigg(self, sch, page=0):
......@@ -498,13 +489,9 @@ class TorrentSearch:
seeds_results = soup.find_all("td", class_="sy")
size_results = soup.find_all("td", class_="size-row")
if sys.version_info > (3, 0):
with ThreadPoolExecutor(max_workers=5) as executor:
for res in executor.map(isohunt_parse_results, zip(row, seeds_results, size_results)):
yield res
else:
for z in zip(row, seeds_results, size_results):
yield(isohunt_parse_results(z))
with ThreadPoolExecutor(max_workers=5) as executor:
for res in executor.map(isohunt_parse_results, zip(row, seeds_results, size_results)):
yield res
def _torrent9(self, sch, page=0):
......@@ -522,46 +509,52 @@ class TorrentSearch:
s = do_req_get(url, "text")
soup = bs4.BeautifulSoup(s, "html.parser")
tmplist = []
name = str()
magnet = str()
size = str()
seeds = str()
leechs = str()
for i in soup.find_all('tbody'):
for d in i.find_all('td'):
if d.find('a'):
href = d.find('a')['href']
# le titre est dans des <span>...
name = [ s.string for s in d.a.find_all('span') ]
name = ' '.join(name)
# get magnet link
ms = do_req_get("{}/{}".format(baseurl,href), "text")
magnetsoup = bs4.BeautifulSoup(ms, "html.parser")
for m in magnetsoup.find_all('a', class_='download'):
if m['href'].startswith('magnet'):
magnet = m['href']
break
toparse = soup.find('tbody')
for d in toparse.find_all('td'):
if d.find('a'):
magnet = d.find('a')['href']
# le titre est dans des <span>...
name = [ s.string for s in d.a.find_all('span') ]
name = ' '.join(name)
else:
if not size:
size = d.string
elif not seeds :
seeds = d.string
elif not leechs :
leechs = d.string
else:
if not size:
size = d.string
elif not seeds :
seeds = d.string
elif not leechs :
leechs = d.string
if name and magnet and size and seeds and leechs :
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
tmplist.append(res)
name = str()
magnet = str()
size = str()
seeds = str()
leechs = str()
if name and magnet and size and seeds and leechs :
with ThreadPoolExecutor(max_workers=5) as executor:
future_mgt = { executor.submit(do_req_get, "{}/{}".format(baseurl,res['magnet']), "text") : res for res in tmplist }
for f in as_completed(future_mgt):
item = future_mgt[f]
ms = f.result()
magnetsoup = bs4.BeautifulSoup(ms, "html.parser")
for m in magnetsoup.find_all('a', class_='download'):
if m['href'].startswith('magnet'):
magnet = m['href']
break
item['magnet'] = magnet
yield item
res = {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
name = str()
magnet = str()
size = str()
seeds = str()
leechs = str()
yield res
def _alphareign(self, sch, page=0):
"""
......@@ -699,7 +692,6 @@ class TorrentSearch:
size : size
seed : number of seeds
"""
page = int(page) + 1
if page < 10:
page = page * 10
......@@ -739,11 +731,6 @@ class TorrentSearch:
yield res
def isohunt_parse_results(z):
"""
parse isohunt results
......@@ -770,13 +757,12 @@ def isohunt_parse_results(z):
return {'name': name, 'magnet': magnet, 'size': size, 'seeds': seeds, "leechs": leechs}
if __name__ == '__main__':
with TorrentSearch('p2psearch') as ts:
s = ts.search('vikings vostfr', 0)
#s = omdb_sch("vikings")
print(s)
with TorrentSearch('torrent9') as ts:
for result in ts.search('vikings vostfr', 0):
print(result)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment