Commit 353c77d0 authored by njean42's avatar njean42
Browse files

Merge small neighbour panels into new, bigger ones. Allows detection of texts...

Merge small neighbour panels into new, bigger ones. Allows detection of texts or drawings outside of frames (regular panel borders).
parent c01eb506
......@@ -4,6 +4,7 @@ import os, json, sys, tempfile, requests
import cv2 as cv
import numpy as np
from urllib.parse import urlparse
from functools import reduce
from lib.panel import Panel
from lib.debug import Debug
......@@ -15,6 +16,8 @@ class NotAnImageException (Exception):
class Kumiko:
DEFAULT_MIN_PANEL_SIZE_RATIO = 1/15
options = {}
img = False
......@@ -28,7 +31,7 @@ class Kumiko:
if self.options['rtl']:
Panel.set_numbering('rtl')
self.options['min_panel_size_ratio'] = 1/15
self.options['min_panel_size_ratio'] = Kumiko.DEFAULT_MIN_PANEL_SIZE_RATIO
if 'min_panel_size_ratio' in options and options['min_panel_size_ratio']:
self.options['min_panel_size_ratio'] = options['min_panel_size_ratio']
......@@ -104,12 +107,56 @@ class Kumiko:
else:
raise Exception('Fatal error, unknown background color: '+str(bgcol))
self.dbg.write_image(thresh,filename+'-020-thresh[bg:{}].jpg'.format(bgcol))
self.dbg.write_image(thresh,filename+'-020-thresholded[bg:{}].jpg'.format(bgcol))
return contours
def split_panels(self,panels,img,contourSize):
def group_small_panels(self, panels, filename, contourSize):
g = 0
groups = {} # panel: groups for this panel
for p1 in panels:
if not p1.is_small():
continue
for p2 in panels:
if p2 == p1 or not p2.is_small():
continue
if p2.is_close(p1):
if not (p1 in groups): groups[p1] = set()
if not (p2 in groups): groups[p2] = set()
grps = groups[p1].union(groups[p2])
use_group = g
if len(grps) > 0:
use_group = min(grps)
else:
g += 1
groups[p1].add(use_group)
groups[p2].add(use_group)
panels = set(panels)
for g in range(g):
panels_in_group = {p:grps for (p,grps) in groups.items() if g in grps} # "dictionary comprehension"
merged_big_panel = reduce(Panel.merge, panels_in_group.keys())
# add big panel and remove small ones
if not merged_big_panel.is_small():
panels.add(merged_big_panel)
tmp_img = self.dbg.draw_panels(self.img,panels_in_group,contourSize,Debug.colours['lightblue'])
tmp_img = self.dbg.draw_panels(tmp_img,[merged_big_panel],contourSize,Debug.colours['green'])
self.dbg.write_image(tmp_img, filename+'-035-merged-small-panels[group{}].jpg'.format(g))
panels = panels - panels_in_group.keys()
panels = list(filter(lambda p: not p.is_small(), panels)) # also remove small panels that were not part of groups
return panels
def split_panels(self,panels,contourSize):
new_panels = []
old_panels = []
for p in panels:
......@@ -118,7 +165,7 @@ class Kumiko:
old_panels.append(p)
new_panels += new
self.dbg.write_contours(img, list(map(lambda n: n.polygon, new)), contourSize)
self.dbg.draw_contours(self.img, list(map(lambda n: n.polygon, new)), contourSize)
for p in old_panels:
panels.remove(p)
......@@ -151,8 +198,10 @@ class Kumiko:
for j in range(i+1,len(panels)):
if panels[i].contains(panels[j]):
panels_to_remove.append(j)
panels[i] = Panel.merge(panels[i],panels[j])
elif panels[j].contains(panels[i]):
panels_to_remove.append(i)
panels[j] = Panel.merge(panels[i],panels[j])
for i in reversed(sorted(list(set(panels_to_remove)))):
del panels[i]
......@@ -214,6 +263,8 @@ class Kumiko:
'filename': url if url else os.path.basename(filename),
'size': size
}
Panel.img_size = size
Panel.small_panel_ratio = self.options['min_panel_size_ratio']
# get license for this file
if os.path.isfile(filename+'.license'):
......@@ -225,6 +276,7 @@ class Kumiko:
sys.exit(1)
self.gray = cv.cvtColor(self.img,cv.COLOR_BGR2GRAY)
self.dbg.write_image(self.gray, filename+'-010-grayed.jpg')
for bgcol in ['white','black']:
res = self.parse_image_with_bgcol(infos.copy(),filename,bgcol,url)
......@@ -247,25 +299,36 @@ class Kumiko:
epsilon = 0.001 * arclength
approx = cv.approxPolyDP(contour,epsilon,True)
panel = Panel(polygon=approx)
# exclude very small panels
if panel.w < infos['size'][0] * self.options['min_panel_size_ratio'] or panel.h < infos['size'][1] * self.options['min_panel_size_ratio']:
continue
self.dbg.write_contours(self.img, [approx], contourSize, colour=(0,0,255))
self.dbg.draw_contours(self.img, [approx], contourSize, Debug.colours['red'])
panels.append(Panel(polygon=approx))
self.dbg.write_image(self.img, filename+'-030-initial-contours.jpg')
# Group small panels that are close together, into bigger ones
panels = self.group_small_panels(panels,filename, contourSize)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-040-merged-small-panels.jpg')
# See if panels can be cut into several (two non-consecutive points are close)
self.split_panels(panels,self.img,contourSize)
self.split_panels(panels,contourSize)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-050-contours-split-panels.jpg')
# Merge panels that shouldn't have been split (speech bubble diving in a panel)
self.merge_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-060-merged-all-panels.jpg')
# splitting polygons may result in panels slightly overlapping, de-overlap them
self.deoverlap_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-070-deoverlaped-panels.jpg')
# re-filter out small panels
panels = list(filter(lambda p: not p.is_small(), panels))
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-080-excluded-small-panels.jpg')
# get actual gutters before expanding panels
actual_gutters = Kumiko.actual_gutters(panels)
infos['gutters'] = [actual_gutters['x'],actual_gutters['y']]
......@@ -273,6 +336,8 @@ class Kumiko:
panels.sort() # TODO: remove
self.expand_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-090-expanded-panels.jpg')
if len(panels) == 0:
panels.append( Panel([0,0,infos['size'][0],infos['size'][1]]) );
......@@ -283,9 +348,4 @@ class Kumiko:
panels = list(map(lambda p: p.to_xywh(), panels))
infos['panels'] = panels
# write panel numbers on debug image
self.dbg.write_image(self.gray, filename+'-010-gray.jpg')
self.dbg.write_image(self.img, filename+'-040-contours.jpg')
return infos
......@@ -6,6 +6,19 @@ import cv2 as cv
class Debug:
colours = {
'white': (255,255,255),
'red': (0,0,255),
'green': (0,255,0),
'blue': (255,0,0),
'lightblue': (200,200,0),
'lightpurple': (200,0,200),
'yellow': (0,200,200),
'gray': (150,150,150),
}
subpanel_colours = list(colours.values())[3:] # white, red and green are used to display other infos
def __init__(self, debug_dir):
self.debug_dir = debug_dir
......@@ -17,9 +30,7 @@ class Debug:
cv.imwrite(os.path.join(self.debug_dir, os.path.basename(filename)),img)
subpanel_colours = [(0,255,0),(255,0,0),(200,200,0),(200,0,200),(0,200,200),(150,150,150)]
def write_contours(self, img, contours, contourSize, colour='auto'):
def draw_contours(self, img, contours, contourSize, colour='auto'):
if not self.debug_dir:
return
......@@ -28,3 +39,19 @@ class Debug:
colour = Debug.subpanel_colours[i % len(Debug.subpanel_colours)]
cv.drawContours(img, [contours[i]], 0, colour, contourSize)
def draw_panels(self, img, panels, contourSize, colour):
if not self.debug_dir:
return
img = img.copy()
for p in panels:
cv.rectangle(img, (p.x,p.y), (p.r,p.b), colour, contourSize)
# + draw inner white border
for p in panels:
cv.rectangle(img, (p.x+contourSize,p.y+contourSize), (p.r-contourSize,p.b-contourSize), Debug.colours['white'], int(contourSize/2))
return img
......@@ -14,7 +14,7 @@ class HTML:
<script type="text/javascript" src="{reldir}reader.js"></script>
<link rel="stylesheet" media="all" href="{reldir}style.css" />
<style type="text/css">
h2 {{ text-align: center; }}
h2 {{ text-align: center; margin-top: 3em; }}
.sidebyside {{ display: flex; justify-content: space-around; }}
.sidebyside > div {{ width: 45%; }}
.version {{ text-align: center; }}
......
......@@ -7,6 +7,8 @@ import numpy as np
class Panel:
img_size = None
small_panel_ratio = None
numbering = 'ltr' # left-to-right by default
def set_numbering(numbering):
......@@ -14,7 +16,14 @@ class Panel:
raise Exception('Fatal error, unknown numbering: '+str(numbering))
Panel.numbering = numbering
def __init__(self, xywh=None, polygon=None):
if Panel.img_size is None:
raise Exception("Fatal error: Panel.img_size hasn't been set")
if Panel.small_panel_ratio is None:
raise Exception("Fatal error: Panel.small_panel_ratio hasn't been set")
if xywh is None and polygon is None:
raise Exception('Fatal error: no parameter to define Panel boundaries')
......@@ -64,6 +73,7 @@ class Panel:
def area(self): return self.w * self.h
def __str__(self): return "[left: {}, right: {}, top: {}, bottom: {} ({}x{})]".format(self.x,self.r,self.y,self.b,self.w,self.h)
def __hash__(self): return hash(self.__str__())
def __setattr__(self,name,value):
if name in ['w','h','wt','ht']:
......@@ -78,6 +88,11 @@ class Panel:
super().__setattr__('ht',self.h / 10) # ht = height threshold
def is_small(self):
return self.w < Panel.img_size[0] * Panel.small_panel_ratio or \
self.h < Panel.img_size[1] * Panel.small_panel_ratio
def overlap_panel(self,other):
if self.x > other.r or other.x > self.r: # panels are left and right from one another
return None
......@@ -98,8 +113,8 @@ class Panel:
if not o_panel:
return False
# self contains other if their overlapping area is more than 75% of other's area
return o_panel.area() / other.area() > 0.75
# self contains other if their overlapping area is more than 50% of other's area
return o_panel.area() / other.area() > 0.50
def same_row(self,other): return other.y <= self.y <= other.b or self.y <= other.y <= self.b
......@@ -135,9 +150,23 @@ class Panel:
}[d](panels)
def merge(p1,p2):
min_x = min(p1.x, p2.x)
min_y = min(p1.y, p2.y)
max_r = max(p1.r, p2.r)
max_b = max(p1.b, p2.b)
return Panel([min_x, min_y, max_r - min_x, max_b - min_y])
def is_close(self, other):
bounding_box = Panel.merge(self,other)
return (self.w + other.w)*2 > bounding_box.w and \
(self.h + other.h)*2 > bounding_box.h
def split(self):
if self.polygon is None:
raise Exception('Fatal error, trying to split a Panel with no polygon (not the result of opencv.findContours)')
return None
close_dots = []
for i in range(len(self.polygon)-1):
......
......@@ -12,8 +12,8 @@ class Tester:
files = []
git_repo = 'https://framagit.org/nicooo/kumiko'
git_versions = [
# 'v1.0', 'v1.1', 'v1.1.1', 'v1.2',
'v1.2.1',
# 'v1.0', 'v1.1', 'v1.1.1', 'v1.2', 'v1.2.1',
'v1.3',
'current',
]
......@@ -41,7 +41,7 @@ class Tester:
def run(self, git_version):
print('\n########## Finding speech bubbles with kumiko version',git_version,'##########')
print('\n########## Finding panels with kumiko version',git_version,'##########')
kumiko_bin = './kumiko'
# Kumiko would not accept non-image files before v1.2, special case for .licence files
......@@ -106,6 +106,9 @@ class Tester:
print('error, image sizes are not the same',json1[p]['size'],json2[p]['size'])
continue
Panel.img_size = json1[p]['size']
Panel.small_panel_ratio = Kumiko.DEFAULT_MIN_PANEL_SIZE_RATIO
panels_v1 = list(map(lambda p: Panel(p), json1[p]['panels']))
panels_v2 = list(map(lambda p: Panel(p), json2[p]['panels']))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment