Commit 19db2262 authored by haberman's avatar haberman

rejects non cp1252 tweets and cleans print

parent 68b318c6
No preview for this file type
......@@ -11,6 +11,7 @@ from . import private
from poemade import guess_form
from poemade.tokenizer import tokenize
from poemade.parser import to_cp1252
QUERY_CREATE_HISTORY = '''CREATE TABLE IF NOT EXISTS history
......@@ -89,7 +90,6 @@ def twitter_statuses_fmt(statuses, min_char):
if len(words) == 0:
del tokens[i]
for status in statuses:
st_text = status.full_text
if status.retweeted_status:
......@@ -113,7 +113,11 @@ def twitter_statuses_fmt(statuses, min_char):
if len(st_text) <= min_char:
continue
s_form = guess_form(tokenize(st_text))
st_text=to_cp1252(st_text)
if not st_text:
continue
s_form=guess_form(tokenize(st_text))
del s_form['stanza']
del s_form['metric']
......@@ -124,7 +128,7 @@ def twitter_statuses_fmt(statuses, min_char):
def twitter_search(q, lang, min_char):
q = 'q={}&lang={}&result_type=recent&count=20'.format(quote(q), lang)
statuses = twitter.GetSearch(raw_query=q)
q='q={}&lang={}&result_type=recent&count=20'.format(quote(q), lang)
statuses=twitter.GetSearch(raw_query = q)
return twitter_statuses_fmt(statuses, min_char)
......@@ -62,9 +62,7 @@ def cbk_release():
print('Inserting {} statuses'.format(len(poems)))
db_rows = ['status_id', 'status_author', 'status_content', 'status_info', 'status_hashtags']
res = db.add_multiple('statuses', poems, rows=db_rows)
if res:
print('OK')
db.add_multiple('statuses', poems, rows=db_rows)
def prepare():
......
......@@ -201,9 +201,6 @@ def rhyme_scheme(tokenized_poem):
scheme[l] = scheme[i] = rhyme_notation[currrhyme].upper()
else:
scheme[l] = scheme[i] = rhyme_notation[currrhyme]
# else:
# print('cant find a rhyme for: {}, {}'.format(base_line[-1], current_line[-1]))
return scheme
......
......@@ -37,11 +37,10 @@ def combine_twitter(source, candidates, idx=0):
c_forms = [(i, json.loads(c[4])) for i, c in enumerate(candidates)]
c_forms.sort(key=lambda cf: distance(s_shape['form']['scheme'], cf[1]['form']['scheme']))
c_idx = c_forms[0][0]
candidate = candidates[c_idx]
print('twitter candidate: {}'.format(candidate))
return s_verses, candidate
......@@ -54,7 +53,6 @@ def combine_gutenberg(source, candidates, idx=0):
'''
def contiguous_range(ints):
print(ints)
ranges = []
for gb in groupby(enumerate(ints), lambda x: x[0]-x[1]):
group = map(itemgetter(1), gb[1])
......@@ -74,7 +72,6 @@ def combine_gutenberg(source, candidates, idx=0):
ss, st, sv = s_shape, s_tokens, s_verses
cs, ct, cv = c_shape, c_tokens, c_verses
if len(c_tokens) > len(s_tokens):
print('reversing')
ss, st, sv = c_shape, c_tokens, c_verses
cs, ct, cv = s_shape, s_tokens, s_verses
s_shape, s_tokens, s_verses = ss, st, sv
......@@ -140,15 +137,7 @@ def combine_gutenberg(source, candidates, idx=0):
if len(c_srhymes) > 0:
print('got crossing rhymes, should check when they occured')
print(c_srhymes)
# c_contigs = contiguous_tuples(c_srhymes)
# s_contigs = contiguous_tuples(c_contigs)
# if len(s_contigs) > 0 and len(c_contigs) > 0:
# print('got contiguity')
# print(s_contigs)
# print(c_contigs)
elif lines_count > 10:
print('got to figure out something creative')
elif lines_count > 8:
c_pron = c_shape['form']['scheme'].split(',')
s_pron = s_shape['form']['scheme'].split(',')
s_cprons = []
......@@ -189,21 +178,12 @@ def combine_gutenberg(source, candidates, idx=0):
combine_done = True
if not combine_done:
print('last attempt to do something, simple buthery for now')
print('last attempt to do something, simple butchery for now')
c_range = range(c_range.start, c_range.start+4)
s_range = range(s_range.start, s_range.start+4)
print(s_cprons)
# c_range = range(c_range.start, c_range.start+4)
# s_range = range(s_range.start, s_range.start+4)
print(s_rhymes)
print(c_rhymes)
return c_bookid, c_stzidx, s_verses[s_range.start:s_range.stop], c_verses[c_range.start:c_range.stop]
# TODO @incomplete -> the stz_info can be of type 'unknown', we should do a lev distance on scheme to sort the closest stanza
def stanza_sort(source, candidates):
'''Returns a list of dict sorting stanzas being the best candidates to answer a source poem.
......
......@@ -61,13 +61,8 @@ def is_poem(verses, line_size=VERSE_MAX):
para_score = 1
inc_score = 1/line_size
# tokens = tokenize('\n'.join(verses), True)
# diversity = lexical_diversity(tokens)
# print(tokens)
# print(diversity)
for v in verses:
line_checks = {k: f(v.strip()) for k, f in poetry_checks.items()}
# print(line_checks)
if all(line_checks):
poem_score += inc_score
elif len(v) > line_size:
......@@ -79,5 +74,4 @@ def is_poem(verses, line_size=VERSE_MAX):
poem_score -= inc_score
para_score += inc_score
# print(poem_score, para_score)
return poem_score > para_score
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment