Commit 65b2f374 authored by njean42's avatar njean42
Browse files

Better debugging.

parent 2e7da8e4
......@@ -25,7 +25,7 @@ An HTML file is generated under `./tests/results/` that you can open locally (ad
It will show you all the differences between the previous git version and yours!
Take a look at the differences between *Kumiko* versions:
* [v1.2.1 and v1.3](https://kumiko.njean.me/tests/results/diff-v1.2.1-v1.3.html)
* [v1.3 and v1.2.1](https://kumiko.njean.me/tests/results/diff-v1.2.1-v1.3.html)
* [v1.2.1 and v1.2](https://kumiko.njean.me/tests/results/diff-v1.2-v1.2.1.html)
* [v1.2 and v1.1](https://kumiko.njean.me/tests/results/diff-v1.1-v1.2.html)
* [v1.1 and v1.0](https://kumiko.njean.me/tests/results/diff-v1.0-v1.1.html)
......@@ -40,7 +40,7 @@ The 4 values representing a panel are given in the following order: **[x,y,width
Panels are sorted in left-to-right reading order by default.
Add the `--rtl` option to have them sorted in right-to-left reading order (e.g. mangas).
Note that, although Kumiko can detect panels with arbitrary shapes, it will return a rectangular bounding box for each one.
Note that, although *Kumiko* can detect panels with arbitrary shapes, it will return a rectangular bounding box for each one.
This is because in the end, many things are rectangular: image files, our screens to display them...
We may want an option in the future to get more accurate polygons instead.
......@@ -78,11 +78,9 @@ The image order is alphabetical, which is based on a commonly encountered *page0
## Debug
You can pass `kumiko` a `--debug-dir /path/to/debug-info` parameter that tells you are craving debugging information.
You can pass `kumiko` a `--debug` parameter that tells you are craving debugging information.
This will create several images, for each page that Kumiko analysed, named `$filename-$step-$operation.jpg`.
These files will be found in the `/path/to/debug-info` folder (should exist and be writable), and give an idea of all `$step`s *Kumiko* is taking to detect panels.
Depending on the `$operation` performed right before creating this file, it will display contours and/or panels on each image.
This will create an HTML file (add `-b firefox` to open it automatically) displaying the various processing steps *Kumiko* has taken, and the panels discovered, added, deleted or rearranged at each stage.
## License information
......
......@@ -22,14 +22,14 @@ parser.add_argument('--html-static-dir', nargs=1, help='When generating HTML, th
parser.add_argument('--min-panel-size-ratio', nargs=1, type=float, help='Panels will be considered too small and exluded if they have a width < img.width * ratio or height < img/height * ratio (default is 1/15th)')
# Utilities
parser.add_argument('--debug-dir', nargs=1, help='Write debug files in this directory (image files with extra panel information and more)')
parser.add_argument('--debug', action='store_true', help="Generate an HTML debug file to show Kumiko's processing steps")
parser.add_argument('--progress', action='store_true', help='Prints progress information')
args = parser.parse_args()
k = Kumiko({
'debug_dir': args.debug_dir[0] if args.debug_dir else False,
'debug': args.debug,
'progress': args.progress,
'rtl': args.rtl,
'min_panel_size_ratio': args.min_panel_size_ratio[0] if args.min_panel_size_ratio else False
......@@ -56,6 +56,7 @@ elif len(args.input) == 1 and os.path.isfile(args.input[0]):
# URL list
else:
folder = 'urls'
info = k.parse_url_list(args.input)
filehash = hashlib.sha1(';'.join(sorted(args.input)).encode()).hexdigest()
html_file = os.path.join('tests/results',filehash+'.html')
......@@ -68,17 +69,22 @@ info = json.dumps(info)
# Generate HTML
if args.html or args.browser or args.html_static_dir:
images_dir = os.path.relpath(folder,'tests/results')+'/' if folder else 'urls'
if args.html or args.browser or args.html_static_dir or args.debug:
images_dir = 'urls' if folder == 'urls' else os.path.relpath(folder,'tests/results')+'/'
reldir = args.html_static_dir[0] if args.html_static_dir else '../../'
html = ''
html += HTML.header(reldir = args.html_static_dir[0] if args.html_static_dir else '../../')
html += HTML.reader(info,images_dir)
html += HTML.footer
if args.debug:
html = k.dbg.html(folder,reldir)
else:
html += HTML.header(reldir = reldir)
html += HTML.reader(info,images_dir)
html += HTML.footer
if args.output:
html_file = args.output[0]
if args.browser or args.output:
if args.browser or args.output or args.debug:
fh = open(html_file, 'w')
fh.write(html)
fh.close()
......@@ -96,6 +102,6 @@ else:
print(info)
# Open in browser
if args.browser:
subprocess.run([args.browser[0],html_file])
......@@ -16,14 +16,12 @@ class NotAnImageException (Exception):
class Kumiko:
DEFAULT_MIN_PANEL_SIZE_RATIO = 1/15
options = {}
img = False
def __init__(self,options={}):
self.dbg = Debug(options['debug_dir'] if 'debug_dir' in options else False)
self.dbg = Debug('debug' in options and options['debug'])
for o in ['progress','rtl']:
self.options[o] = o in options and options[o]
......@@ -31,7 +29,7 @@ class Kumiko:
if self.options['rtl']:
Panel.set_numbering('rtl')
self.options['min_panel_size_ratio'] = Kumiko.DEFAULT_MIN_PANEL_SIZE_RATIO
self.options['min_panel_size_ratio'] = Panel.DEFAULT_MIN_PANEL_SIZE_RATIO
if 'min_panel_size_ratio' in options and options['min_panel_size_ratio']:
self.options['min_panel_size_ratio'] = options['min_panel_size_ratio']
......@@ -107,12 +105,12 @@ class Kumiko:
else:
raise Exception('Fatal error, unknown background color: '+str(bgcol))
self.dbg.write_image(thresh,filename+'-020-thresholded[bg:{}].jpg'.format(bgcol))
self.dbg.add_image(thresh,'Thresholded image, supposed {} background'.format(bgcol))
return contours
def group_small_panels(self, panels, filename, contourSize):
def group_small_panels(self, panels, filename):
g = 0
groups = {} # panel: groups for this panel
for p1 in panels:
......@@ -146,17 +144,20 @@ class Kumiko:
if not merged_big_panel.is_small():
panels.add(merged_big_panel)
tmp_img = self.dbg.draw_panels(self.img,panels_in_group,contourSize,Debug.colours['lightblue'])
tmp_img = self.dbg.draw_panels(tmp_img,[merged_big_panel],contourSize,Debug.colours['green'])
self.dbg.write_image(tmp_img, filename+'-035-merged-small-panels[group{}].jpg'.format(g))
tmp_img = self.dbg.draw_panels(self.img, panels_in_group, Debug.colours['lightblue'])
tmp_img = self.dbg.draw_panels(tmp_img, [merged_big_panel], Debug.colours['green'])
self.dbg.add_image(tmp_img, 'Group small panels')
panels = panels - panels_in_group.keys()
panels = list(filter(lambda p: not p.is_small(), panels)) # also remove small panels that were not part of groups
self.dbg.add_step('Group small panels', panels)
return panels
def split_panels(self,panels,contourSize):
def split_panels(self,panels):
new_panels = []
old_panels = []
for p in panels:
......@@ -165,11 +166,18 @@ class Kumiko:
old_panels.append(p)
new_panels += new
self.dbg.draw_contours(self.img, list(map(lambda n: n.polygon, new)), contourSize)
self.dbg.draw_contours(self.img, list(map(lambda n: n.polygon, new)))
for p in old_panels:
panels.remove(p)
panels += list(filter(lambda p: not p.is_small(), new_panels))
panels += new_panels
self.dbg.add_image(self.img, 'Split contours')
self.dbg.add_step('Panels from split contours', panels)
panels = list(filter(lambda p: not p.is_small(), panels))
self.dbg.add_step('Exclude small panels', panels)
def deoverlap_panels(self,panels):
......@@ -189,6 +197,8 @@ class Kumiko:
panels[i].b = opanel.y
panels[j].y = opanel.b
continue
self.dbg.add_step('Deoverlap panels', panels)
# Merge every two panels where one contains the other
......@@ -205,6 +215,8 @@ class Kumiko:
for i in reversed(sorted(list(set(panels_to_remove)))):
del panels[i]
self.dbg.add_step('Merge panels', panels)
# Find out actual gutters between panels
......@@ -229,7 +241,7 @@ class Kumiko:
}
# Expand panels to their neighbour's edge, or page frame
# Expand panels to their neighbour's edge, or page boundaries
def expand_panels(self, panels):
gutters = Kumiko.actual_gutters(panels)
for i in range(len(panels)):
......@@ -249,6 +261,8 @@ class Kumiko:
if newcoord != -1:
if d in ['r','b'] and newcoord > getattr(panels[i],d) or d in ['x','y'] and newcoord < getattr(panels[i],d):
setattr(panels[i],d,newcoord)
self.dbg.add_step('Expand panels', panels)
def parse_image(self,filename,url=None):
......@@ -276,7 +290,7 @@ class Kumiko:
sys.exit(1)
self.gray = cv.cvtColor(self.img,cv.COLOR_BGR2GRAY)
self.dbg.write_image(self.gray, filename+'-010-grayed.jpg')
self.dbg.add_image(self.gray,'Shades of gray')
for bgcol in ['white','black']:
res = self.parse_image_with_bgcol(infos.copy(),filename,bgcol,url)
......@@ -284,64 +298,56 @@ class Kumiko:
return res
return res
def parse_image_with_bgcol(self,infos,filename,bgcol,url=None):
contours = self.get_contours(self.gray,filename,bgcol)
infos['background'] = bgcol
self.dbg.infos = infos.copy()
# Get (square) panels out of contours
contourSize = int(sum(infos['size']) / 2 * 0.004)
self.dbg.contourSize = int(sum(infos['size']) / 2 * 0.004)
panels = []
for contour in contours:
arclength = cv.arcLength(contour,True)
epsilon = 0.001 * arclength
approx = cv.approxPolyDP(contour,epsilon,True)
self.dbg.draw_contours(self.img, [approx], contourSize, Debug.colours['red'])
self.dbg.draw_contours(self.img, [approx], Debug.colours['red'])
panels.append(Panel(polygon=approx))
self.dbg.write_image(self.img, filename+'-030-initial-contours.jpg')
self.dbg.add_image(self.img, 'Initial contours')
self.dbg.add_step('Panels from initial contours', panels)
# Group small panels that are close together, into bigger ones
panels = self.group_small_panels(panels,filename, contourSize)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-040-merged-small-panels.jpg')
panels = self.group_small_panels(panels,filename)
# See if panels can be cut into several (two non-consecutive points are close)
self.split_panels(panels,contourSize)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-050-contours-split-panels.jpg')
self.split_panels(panels)
# Merge panels that shouldn't have been split (speech bubble diving in a panel)
self.merge_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-060-merged-all-panels.jpg')
# splitting polygons may result in panels slightly overlapping, de-overlap them
self.deoverlap_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-070-deoverlaped-panels.jpg')
# re-filter out small panels
panels = list(filter(lambda p: not p.is_small(), panels))
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-080-excluded-small-panels.jpg')
self.dbg.add_step('Exclude small panels', panels)
# get actual gutters before expanding panels
actual_gutters = Kumiko.actual_gutters(panels)
infos['gutters'] = [actual_gutters['x'],actual_gutters['y']]
panels.sort() # TODO: remove
panels.sort() # TODO: remove when panels expansion is smarter
self.expand_panels(panels)
self.dbg.write_image(self.dbg.draw_panels(self.img,panels,contourSize,Debug.colours['green']), filename+'-090-expanded-panels.jpg')
if len(panels) == 0:
panels.append( Panel([0,0,infos['size'][0],infos['size'][1]]) );
# Number panels comics-wise (left to right for now)
# Number panels comics-wise (ltr/rtl)
panels.sort()
# Simplify panels back to lists (x,y,w,h)
......
import os
import os, json
import cv2 as cv
from lib.html import HTML
from lib.panel import Panel
class Debug:
......@@ -16,42 +19,158 @@ class Debug:
'yellow': (0,200,200),
'gray': (150,150,150),
}
subpanel_colours = list(colours.values())[3:] # white, red and green are used to display other infos
subpanel_colours = list(colours.values())[3:] # white, red and green are used to display main and split panels
def __init__(self, debug_dir):
self.debug_dir = debug_dir
def __init__(self, debug):
self.debug = debug
self.contourSize = None
self.steps = []
self.images = {}
self.infos = {}
def write_image(self, img, filename):
if not self.debug_dir:
def add_step(self, name, panels):
if not self.debug:
return
cv.imwrite(os.path.join(self.debug_dir, os.path.basename(filename)),img)
self.steps.append({
'name': name,
'panels': panels.copy()
})
imgID = 0
def add_image(self,img,label):
if not self.debug:
return
currstep = 'init'
if len(self.steps) > 0:
currstep = list(reversed(self.steps))[0]['name']
filename = str(Debug.imgID) + label+'.jpg'
Debug.imgID += 1
cv.imwrite(os.path.join('tests/results',filename),img)
if not(currstep in self.images):
self.images[currstep] = []
self.images[currstep].append({'filename': filename, 'label': label})
def html(self, images_dir, reldir):
html = ''
html += HTML.header(title='Debugging - Kumiko processing steps', reldir=reldir)
for i in range(len(self.steps)-1):
j = i + 1
# Display debug images
if i == 0:
html += HTML.imgbox(self.images['init'])
if self.steps[i]['name'] in self.images:
html += HTML.imgbox(self.images[self.steps[i]['name']])
# Display panels diffs
self.infos['panels'] = list(map(lambda p: p.to_xywh(), self.steps[i]['panels']))
json1 = self.infos.copy()
self.infos['panels'] = list(map(lambda p: p.to_xywh(), self.steps[j]['panels']))
json2 = self.infos.copy()
files_diff = Debug.get_files_diff(images_dir, [json1], [json2])
step_name = str(i+1) + '. ' +self.steps[j]['name']
if len(files_diff) == 0:
html += "<h2>{} - no change</h2>".format(step_name);
for filename in files_diff:
html += HTML.side_by_side_panels(
step_name,
files_diff[filename]['jsons'],
'BEFORE - {} panels'.format(len(files_diff[filename]['jsons'][0][0]['panels'])),
'AFTER - {} panels'.format(len(files_diff[filename]['jsons'][1][0]['panels'])),
images_dir=files_diff[filename]['images_dir'],
known_panels=files_diff[filename]['known_panels'],
)
html += HTML.footer
return html
def get_files_diff(file_or_dir,json1,json2):
files_diff = {}
for p in range(len(json1)): # for each page
# check both images' filename and size, should be the same
if os.path.basename(json1[p]['filename']) != os.path.basename(json2[p]['filename']):
print('error, filenames are not the same',json1[p]['filename'],json2[p]['filename'])
continue
if json1[p]['size'] != json2[p]['size']:
print('error, image sizes are not the same',json1[p]['size'],json2[p]['size'])
continue
Panel.img_size = json1[p]['size']
Panel.small_panel_ratio = Panel.DEFAULT_MIN_PANEL_SIZE_RATIO
panels_v1 = list(map(lambda p: Panel(p), json1[p]['panels']))
panels_v2 = list(map(lambda p: Panel(p), json2[p]['panels']))
known_panels = [[],[]]
j = -1
for p1 in panels_v1:
j += 1
if p1 in panels_v2:
known_panels[0].append(j)
j = -1
for p2 in panels_v2:
j += 1
if p2 in panels_v1:
known_panels[1].append(j)
images_dir = 'urls'
if file_or_dir != 'urls':
images_dir = file_or_dir if os.path.isdir(file_or_dir) else os.path.dirname(file_or_dir)
images_dir = os.path.relpath(images_dir,'tests/results')+'/'
if len(known_panels[0]) != len(panels_v1) or len(known_panels[1]) != len(panels_v2):
files_diff[json1[p]['filename']] = {
'jsons': [[json1[p]],[json2[p]]],
'images_dir': images_dir,
'known_panels': [json.dumps(known_panels[0]),json.dumps(known_panels[1])]
}
return files_diff
def draw_contours(self, img, contours, contourSize, colour='auto'):
if not self.debug_dir:
def draw_contours(self, img, contours, colour='auto'):
if not self.debug:
return
if self.contourSize is None:
raise Exception("Fatal error, Debug.contourSize has not been defined");
for i in range(len(contours)):
if colour == 'auto':
colour = Debug.subpanel_colours[i % len(Debug.subpanel_colours)]
cv.drawContours(img, [contours[i]], 0, colour, contourSize)
cv.drawContours(img, [contours[i]], 0, colour, self.contourSize)
def draw_panels(self, img, panels, contourSize, colour):
if not self.debug_dir:
def draw_panels(self, img, panels, colour):
if not self.debug:
return
if self.contourSize is None:
raise Exception("Fatal error, Debug.contourSize has not been defined");
img = img.copy()
for p in panels:
cv.rectangle(img, (p.x,p.y), (p.r,p.b), colour, contourSize)
cv.rectangle(img, (p.x,p.y), (p.r,p.b), colour, self.contourSize)
# + draw inner white border
for p in panels:
cv.rectangle(img, (p.x+contourSize,p.y+contourSize), (p.r-contourSize,p.b-contourSize), Debug.colours['white'], int(contourSize/2))
cv.rectangle(img, (p.x+self.contourSize,p.y+self.contourSize), (p.r-self.contourSize,p.b-self.contourSize), Debug.colours['white'], int(self.contourSize/2))
return img
import json
class HTML:
def header(title='',reldir=''):
......@@ -14,12 +16,12 @@ class HTML:
<script type="text/javascript" src="{reldir}reader.js"></script>
<link rel="stylesheet" media="all" href="{reldir}style.css" />
<style type="text/css">
h2 {{ text-align: center; margin-top: 3em; }}
h2, h3 {{ text-align: center; margin-top: 3em; }}
.sidebyside {{ display: flex; justify-content: space-around; }}
.sidebyside > div {{ width: 45%; }}
.version {{ text-align: center; }}
.kumiko-reader {{ height: 90vh; }}
.kumiko-reader.fullpage {{ height: 100%; width: 100%; }}
.kumiko-reader.halfwidth {{ max-width: 45%; }}
.kumiko-reader.fullpage {{ width: 100%; height: 100%; }}
</style>
</head>
......@@ -34,11 +36,11 @@ class HTML:
pageId = 0
def side_by_side_panels(img,jsons,v1,v2,images_dir,known_panels):
html = '<h2>{0}</h2><div class="sidebyside"><div class="version">{1}</div><div class="version">{2}</div></div><div class="sidebyside">'.format(img,v1,v2)
def side_by_side_panels(title,jsons,v1,v2,images_dir,known_panels):
html = '<h2>{0}</h2><div class="sidebyside"><div class="version">{1}</div><div class="version">{2}</div></div><div class="sidebyside">'.format(title,v1,v2)
oneside = """
<div id="page{id}" class="kumiko-reader debug"></div>
<div id="page{id}" class="kumiko-reader halfwidth debug"></div>
<script type="text/javascript">
var reader = new Reader({{
container: $('#page{id}'),
......@@ -52,13 +54,21 @@ class HTML:
i = -1
for js in jsons:
i += 1
html += oneside.format(id=HTML.pageId,json=js,images_dir=images_dir,known_panels=known_panels[i])
html += oneside.format(id=HTML.pageId,json=json.dumps(js),images_dir=images_dir,known_panels=known_panels[i])
HTML.pageId += 1
html += '</div>'
return html
def imgbox(images):
html = "<h3>Debugging images</h3>\n<div class='imgbox'>\n";
for img in images:
html += "\t<div><p>{}</p><img src='{}' /></div>\n".format(img['label'],img['filename'])
return html + "</div>\n\n"
def reader(js,images_dir):
return """
<div id="reader" class="kumiko-reader fullpage"></div>
......
......@@ -7,6 +7,8 @@ import numpy as np
class Panel:
DEFAULT_MIN_PANEL_SIZE_RATIO = 1/15
img_size = None
small_panel_ratio = None
numbering = 'ltr' # left-to-right by default
......
......@@ -117,6 +117,12 @@ class Reader {
$('.pagenb',this.gui).html('page '+(page+1)+' <small>/'+this.comic.length+'</small>')
var imginfo = this.comic[page];
if (!this.gui.hasClass('fullpage'))
this.gui.css({
width: imginfo.size[0],
height: imginfo.size[1],
padding: '2em',
});
var imgurl = this.images_dir == 'urls' ? imginfo.filename : this.images_dir + imginfo.filename.split('/').reverse()[0];
var img = $('<img class="pageimg" src="'+imgurl+'"/>');
......
......@@ -20,8 +20,6 @@ ul {
.kumiko-reader {
z-index: 1;
height: 600px;
width: 600px;
cursor: pointer;
border: 1px dotted blue;
padding: 1px; /* make sure to see panel borders even for pages with no margin (e.g. xkcd) */
......@@ -29,7 +27,7 @@ ul {
position: relative;
}
.pagenb, .burger, .prev, .menu > ul {
.pagenb, .burger, .prev, .menu > ul, .license {
z-index: 20;
padding: 0.1em 0.4em;
background-color: #34495e;
......@@ -101,7 +99,7 @@ ul {
bottom: 0;
right: 15%;
}
.license a {
.license, .license a {
color: #ecf0f1;
}
......@@ -148,3 +146,15 @@ ul {
.kumiko-reader.debug .container:not(.zoomed) .panel span.bottom { position: absolute; bottom: 0; left: 45%; }
.kumiko-reader.debug .container:not(.zoomed) .panel span.left { position: absolute; left: 0; top: 45%; }
.kumiko-reader.debug .container:not(.zoomed) .panel span.right { position: absolute; right: 0; top: 45%; }
/* Debug mode */
.imgbox {
display: flex;
flex-wrap: wrap;
justify-content: space-around;
}
.imgbox > div { width: 25%; }
.imgbox img { width: 100%; border: 1px dotted gray; }
.imgbox p { text-align: center;}
......@@ -2,7 +2,7 @@
import os, subprocess, json, argparse, re, tempfile
from kumikolib import Kumiko,Panel