Si vous constatez la perte de tickets, de commentaires ou de projets, veuillez consulter la page https://status.framasoft.org/incident/626.

...
 
Commits (3)
......@@ -17,7 +17,7 @@ content-types of the dropped parts.
`match all action rewrite "python3 -IOq drop_alternatives.py" continue`
== Tests
Around 30 _doctests_ are included. You can run them with the following command in
Around 40 _doctests_ are included. You can run them with the following command in
the cloned repository root: +
`./test_drop_alternatives.py`
......
......@@ -124,8 +124,12 @@ def drop_alternatives(msg_bytes, debug=0):
def are_idem_txt(part_txt, part_htm, debug=0):
txt_1, txt_2 = get_txt(part_txt, 3000)
htm_1, htm_2 = get_txt(part_htm, 30000)
htm_1, htm_2 = get_txt(part_htm, 40000)
s_1, s_2 = min(len(htm_1), len(txt_1)), min(len(htm_2), len(txt_2))
if s_1 == 0 or s_2 == 0:
return False
idem_ratio_1 = SequenceMatcher(a=htm_1[:s_1], b=txt_1[:s_1]).quick_ratio()
idem_ratio_2 = SequenceMatcher(a=htm_2[-s_2:], b=txt_2[-s_2:]).quick_ratio()
idem_ratio = (idem_ratio_1 + idem_ratio_2) / 2
......@@ -140,7 +144,7 @@ def are_idem_txt(part_txt, part_htm, debug=0):
def put(string, postfix, size):
print(string[:size].ljust(size, '.') + postfix, file=stderr)
if idem_ratio_1 < LIM: # or True:
if idem_ratio_1 < LIM or True:
put('\n' + str(htm_1, errors='replace'), W + ' <H', PUT_htm - 1)
put(str(txt_1, errors='replace'), ' T '+color_ratio(idem_ratio_1)+ir, PUT_txt)
if idem_ratio_2 < LIM: # or True:
......
Return-Path: <root@bbb.globenet.org>
Delivered-To: aaa@aaa.fr
Received: by bbb (fdm 1.9, account "ccc")
with imap (version=TLSv1/SSLv3 DHE-RSA-AES256-GCM-SHA384 256 bits)
Wed, 20 Jun 2018 18:30:09 +0200
Received: by bbb.globenet.org (Postfix, from userid 0)
id 6D1F07661D; Wed, 20 Jun 2018 18:29:52 +0200 (CEST)
Authentication-Results: bbb.globenet.org; dkim=fail
reason="verification failed; insecure key"
header.d=gmail.com header.i=@gmail.com header.b=uzNvhVRk;
dkim-adsp=none (insecure policy); dkim-atps=neutral
X-policyd-weight: using cached result; rate:hard: -7
Received: from wwinf1u11 ([10.223.74.54]) by mwinf5d31 with ME
id 0Ky91y00J1AH1Dm03Ky9yv; Mon, 18 Jun 2018 21:58:09 +0200
X-ME-Helo: wwinf1u11
X-ME-Auth: bWFrYW1hbG9Ab3JhbmdlLmZy
X-ME-Date: Mon, 18 Jun 2018 21:58:09 +0200
X-ME-IP: 10.79.3.106
Received: from mwinf5c43 (mwinf5c43 [10.223.111.93]) by mwinb2h04 with LMTPA;
Mon, 18 Jun 2018 21:41:15 +0200
Date: Mon, 18 Jun 2018 21:58:09 +0200 (CEST)
From: Ccc CCC <ddd@orange.fr>
Reply-To: Ccc CCC <ddd@orange.fr>
To: bbb@bbb.fr
Message-ID: <793248608.18476.1529351889545.JavaMail.www@wwinf1u11>
In-Reply-To: <CABr+A+6a+4h4KtB=FRNSjXxPZCUNTG3fEAEd7hL36RZSoWj2oA@mail.gmail.com>
References: <0c24f466fd77310596cb98101516bee4@zzz>
<1ae85cbdd6d07d961b3dec7c8d532a66@zzz>
<CABr+A+6a+4h4KtB=FRNSjXxPZCUNTG3fEAEd7hL36RZSoWj2oA@mail.gmail.com>
Subject: =?UTF-8?Q?RE:_Re:_Pique-nique_de_No=C3=ABl_du_30_juin!?=
MIME-Version: 1.0
Content-Type: multipart/alternative;
boundary="----=_Part_18475_677372765.1529351889540"
X-Google-Smtp-Source: ADUXVKLfz2U9ienlUuyD4/tmvHc8jcbojgPTPJyBZWdGVmqFFwdMQytb1NXcQgAIvzy731BN5Vl72woy9IyGTTrTwAM=
X-Cache-ID:
Message-Context: email-message
X-bcc: eeeb@orange.fr
X-Cache-Entry:
X-Wum-Nature: EMAIL-NATURE
X-Message-Size:
X-SAVECOPY: true
DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed; d=gmail.com; s=20161025;
h=mime-version:in-reply-to:references:from:date:message-id:subject:to
:cc; bh=gIA0oSzd0jlTQ6j56Qxfegkm449cRUNI4pjJsWYCnms=;
b=uzNvhVRkMhRXzOyasBo34/3hWDRTl8peh0w6hbBUFZWa4s5mq1HfECUyR3g3BTEnTd
F28hJB+e+GNZL8G5fhcrUsoFQNgH5NCM3QFOtPlk0qWN5eezoQ+V0+NZApdNlSjbS1hv
lGnXIm7MRiACRTP9NM4eFKUyRb5yOdnFfjoAluPt5K/uKsWeL76YWQ+05bNUsmolVcJX
7SdvXW6X7lA0WaE1SAcUR1nUditBtHRozVoxXivPsRMEfc7n1OHKZuGAWXtz8SLJ1/W7
nZ6DAWWRfKsczso+vgwm/bmjXdCgSEcZ9PemNZwNjJSkZsErjWyQo2X5bjLXmnT111nH
8ZcQ==
X-ME-engine: default
X-Country-Code:
X-me-spamlevel: not-spam
X-Google-DKIM-Signature: v=1; a=rsa-sha256; c=relaxed/relaxed;
d=1e100.net; s=20161025;
h=x-gm-message-state:mime-version:in-reply-to:references:from:date
:message-id:subject:to:cc;
bh=gIA0oSzd0jlTQ6j56Qxfegkm449cRUNI4pjJsWYCnms=;
b=k6NzFr08opnfovnAq8Tjk5MMF7W7h7Tvt5WZKjJqU/BqEvswo8bY8U34D9gWYLxM6m
tjQX1dj1686rE+vO73mUM6cg8vwY7GyHdeCiOyVWarbunDEytlULCgP3m78qPco1MoAP
s/CHwbSTXTvUTkkIVcmBTizI//+2Uyvcd5G6KqnjWR5/cPo19raTzA1Efo9HOFTXd/Jf
Dfu3QUHLjdaUIRTsTyUy9Cl/E5u5cCRflv0eCPxiqdcSjiuHDsoeOp4VvktHsvDF8lp8
LlKQIihKM1dECjVih1dv9FCT6DAJy2pbTmYvWjXPc8MikyjtnBvVdQoig96Dsp2Peeb3
KeJA==
X-me-spamcause: (-100)(0000)gggruggvucftvghtrhhoucdtuddrgedthedruddtvddgudefjecutefuodetggdotefrodftvfcurfhrohhfihhlvgemucfogfdpggftiffpkfenuceurghilhhouhhtmecugedttdenucesvcftvggtihhpihgvnhhtshculddquddttddmnecujfgurhepjghfhfffkffuvfgtsegrtderredttdejnecuhfhrohhmpefnuhgtihhlvgcuifhonhhnohhrugcuoehluhgtihhlvgdrghhonhhnohhrugesghhmrghilhdrtghomheqnecukfhppedvtdelrdekhedrvdduhedrieeinecurfgrrhgrmhephhgvlhhopehmrghilhdqlhhftddqfheiiedrghhoohhglhgvrdgtohhmpdhinhgvthepvddtledrkeehrddvudehrdeiiedpmhgrihhlfhhrohhmpehluhgtihhlvgdrghhonhhnohhrugesghhmrghilhdrtghomhdprhgtphhtthhopehgrddvlhesohhrrghnghgvrdhfrhenucevlhhushhtvghrufhiiigvpedt
X-Originating-IP: 10.79.3.106
X-Gm-Message-State: APt69E1qwqYkzlQycJDjacc8qLVEhmkdev3Yng1WL+0j4q3AY42/7InI
+gLY6h7rmUIzYl6BwdPsGDW4prCuOorX04jZdtY=
X-ME-IP: 209.85.215.66
X-Sieve: CMU Sieve 2.3
X-National-Code:
X-Wum-ChannelType:
X-Received: by 2002:a2e:1b0a:: with SMTP id
b10-v6mr9002076ljb.76.1529350875318;
Mon, 18 Jun 2018 12:41:15 -0700 (PDT)
X-WUM-FROM: |~|
X-WUM-TO: |~||~|
X-WUM-CC: |~||~||~||~||~||~|
X-WUM-REPLYTO: |~|
x-simple-spam-score: 1
x-simple-spam-log: near time
Content-Length: 3321
------=_Part_18475_677372765.1529351889540
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 7bit
------=_Part_18475_677372765.1529351889540
Content-Type: text/html; charset=UTF-8
Content-Transfer-Encoding: quoted-printable
<!--OMAAIS--><p dir=3D"ltr">Coucou&nbsp; tout le monde,</p>
<p dir=3D"ltr">On fera des petites choses pour l'ap&#233;ro. </p>
<p dir=3D"ltr">Bisous <br>
<br>
Envoy&#233; depuis l'application Mail Orange</p>
<!--/OMAAIS--><br />----------------<br />Le 18/06/2018, =C3=A0=C2=A021:41,=
Lll LLL a =C3=A9crit=C2=A0:<br /><br /><div dir=3D"ltr"><div>Hello =
from la RSY, <br></div><div><br></div><div>Je m&#39;occupe des sauces et pr=
=C3=A9pare un g=C3=A2teau au chocolat! (m=C3=AAme pas besoin de fayoter=
;-))<br></div><div><br></div><div>Bonne semaine et =C3=A0 tr=C3=A8s vite=
,</div><div><br></div><div>fff<br></div></div><div class=3D"gmail_extra"><=
br><div class=3D"gmail_quote">Le 18 juin 2018 =C3=A0 21:25, hhh Mmm =
<span dir=3D"ltr">&lt;<a href=3D"mailto:ggg@ggg.fr" target=3D"_=
blank">ggg@ggg.fr</a>&gt;</span> a =C3=A9crit :<br><blockquote =
class=3D"gmail_quote" style=3D"margin:0 0 0 .8ex;border-left:1px #ccc solid=
;padding-left:1ex">Re!<br>
<br>
Cool! Merci tata pour ta r=C3=A9ponse rapide!<br>
<br>
C&#39;est donc barbecue avec la viande de kkk, du coup chacun peut se posi=
tionner sur:<br>
Ap=C3=A9ro / salades - accompagnement barbecue, sauces / fromage / dessert =
/ boissons<br>
<br>
Moi je veux bien pr=C3=A9parer une salade et apporter une bouteille de vin,=
et du pain!<br>
<br>
Des bisous de la Couaill=C3=A8re<span class=3D"HOEnZb"><font color=3D"#8888=
88"><br>
hhh</font></span><div class=3D"HOEnZb"><div class=3D"h5"><br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
<br>
Le 2018-06-18 18:00, hhh Mmm a =C3=A9crit=C2=A0:<br>
<blockquote class=3D"gmail_quote" style=3D"margin:0 0 0 .8ex;border-left:1p=
x #ccc solid;padding-left:1ex">
Coucou tout le monde!<br>
<br>
J&#39;esp=C3=A8re que vous =C3=AAtes tous en forme, ici le soleil est l=C3=
=A0 et pour un<br>
moment visiblement, c&#39;est parfait, c&#39;est la fin des exams pour moi!=
<br>
<br>
Le pique-nique du 30 juin arrive =C3=A0 grand pas, on s&#39;=C3=A9tait dit =
avec les<br>
filles (les cousines) que pour faciliter l&#39;organisation du repas et<br>
que ce ne soit pas une charge pour ceux qui sont sur place, qu&#39;on<br>
pourrait tous pr=C3=A9parer/acheter quelque chose que nous partagerions...<=
br>
<br>
Qu&#39;en dites-vous?<br>
<br>
Si l&#39;option est retenue je pr=C3=A9pare aussit=C3=B4t un petit sondage =
avec<br>
r=C3=A9partition des mets, histoire qu&#39;on ne se retrouve pas qu&#39;ave=
c des<br>
saucisses, hein!<br>
En parlant de saucisses, on peut aussi pr=C3=A9voir barbecue avec des<br>
salades... on se dit quoi!<br>
<br>
Bon et sinon, message perso de iii =C3=A0 jjj, faut que tu cuisines<br=
>
parce que ce que tu pr=C3=A9pares c&#39;est trop bon! Donc toi t&#39;as pas=
le<br>
droit t&#39;acheter un truc tout pr=C3=AAt! Pour les autres on jugera sur p=
i=C3=A8ce<br>
;)<br>
<br>
J&#39;attends vos retours,<br>
<br>
Bonne fin de journ=C3=A9e et gros bisous,<br>
hhh<br>
</blockquote>
<br>
</div></div></blockquote></div><br></div>
------=_Part_18475_677372765.1529351889540--
......@@ -105,6 +105,8 @@ def test_drop_alternatives(msg_bytes, debug):
text/html
>>> test_drop_alternatives(open('email_test/20180418.eml', 'rb').read(), DEBUG)
multipart/mixed text/plain multipart/external-body image/jpeg multipart/external-body image/jpeg text/plain multipart/external-body image/jpeg multipart/external-body image/jpeg text/plain
>>> test_drop_alternatives(open('email_test/20180620.eml', 'rb').read(), DEBUG)
multipart/alternative text/plain text/html
"""
print(' '.join([p.get_content_type() for p in drop_alternatives(msg_bytes, debug).walk()]))
......
......@@ -12,7 +12,7 @@ from test_drop_alternatives import test_drop_alternatives
CPU: Intel(R) Xeon(R) CPU L5420 @ 2.50GHz
OS : Debian Jessie 8.9
2017-10-03 : 11 tests ; 3.34s ; 3.03ms/t
2017-10-03 : 11 tests ; 3.34s ; 3.03ms/t 100 runs
2017-10-03 : 11 tests ; 2.24s ; 2.06ms/t (regex del spaces ; compares 200c)
2017-10-03 : 13 tests ; 2.75s ; 2.11ms/t (deps on lxml, compares 100c)
2017-10-04 : 16 tests ; 3.56s ; 2.22ms/t (no deps, no decode, 3 junk-regex, compares 45c)
......@@ -55,6 +55,7 @@ Python 3.5.3
2018-04-24 : 37 tests ; 2.79s ; 7.54ms/t (new hyp test on skip treatment trigger)
2018-04-25 : 37 tests ; 2.80s ; 7.56ms/t (in 50%< startswith, append/pop 30%< inser(0,_)/pop(0))
2018-05-16 : 37 tests ; 2.80s ; 7.56ms/t (.lower() each compared text portions)
2018-06-20 : 38 tests ; 1.95s ; 5.13ms/t (don't compare with zero-sized elts, 40k HTML read)
"""
......