Commit 44b86689 authored by Romain Casati's avatar Romain Casati
Browse files

Towards full Python kernel implementation.

parent a1aa938d
......@@ -52,141 +52,6 @@
* Get the URL of Basthon modules dir.
*/
that.basthonModulesRoot = that.basthonRoot + "/modules";
/**
* A separate namespace for packages (module) managment.
* (defined here since we need basthonModulesRoot)
*/
that.packages = (function () {
let pkg = {};
/**
* Available packages in Pyodide.
* :type: set
*/
pkg.pyodide = null;
/**
* Packages not implemented in Pyodide but in Basthon
* (dict pointing to Pypi or internal addresse).
* :type: dict
*/
pkg.internal = {
"turtle": {
path: that.basthonModulesRoot + "/turtle-0.0.1-py3-none-any.whl",
},
"requests": {
path: that.basthonModulesRoot + "/requests-0.0.1-py3-none-any.whl",
},
"proj4py": {
path: that.basthonModulesRoot + "/proj4py-0.0.1-py3-none-any.whl",
deps: ["pkg_resources"],
},
"folium": {
path: "folium", // loaded from PyPi
},
"graphviz": {
path: that.basthonModulesRoot + "/graphviz-0.0.1-py3-none-any.whl",
deps: ["pkg_resources"],
},
"IPython": {
path: that.basthonModulesRoot + "/IPython-0.0.1-py3-none-any.whl",
},
"p5": {
path: that.basthonModulesRoot + "/p5-0.0.1-py3-none-any.whl",
deps: ["pkg_resources"],
},
};
/**
* Union of internal and pyodide packages.
* :type: set
*/
pkg.all = null;
/**
* Packages already loaded.
* :type: set
*/
pkg.loaded = null;
/**
* Init packages lists.
*/
pkg.init = function () {
pkg.pyodide = new Set(Object.keys(pyodide._module.packages.import_name_to_package_name));
pkg.all = new Set([...pkg.pyodide, ...Object.keys(pkg.internal)]);
pkg.loaded = new Set(); // empty (nothing loaded)
};
/**
* Processing packages before loading
* (common part of Pyodide/internal loading).
*/
pkg._processPackagesBeforeLoad = function (packages) {
if( typeof packages === "string" ) {
packages = [packages];
}
// remove already loaded
packages = packages.filter(p => !pkg.loaded.has(p));
// updating loaded list
packages.forEach(p => pkg.loaded.add(p));
return packages
};
/**
* Loading Pyodide packages.
* Callback function is called on not already loaded packages.
*/
pkg.loadPyodide = async function (packages, callback) {
packages = pkg._processPackagesBeforeLoad(packages);
if( packages.length === 0 ) { return; }
// from Python name to Pyodide name
const pyodidePackages = packages.map(
p => pyodide._module.packages.import_name_to_package_name[p]);
await pyodide.loadPackage(pyodidePackages);
if(callback) {
callback(packages);
}
};
/**
* Loading internal module with micropip (async).
* Callback function is called on not already loaded packages.
*/
pkg.loadInternal = async function (packages, callback) {
packages = pkg._processPackagesBeforeLoad(packages);
if( packages.length === 0 ) { return; }
const packagesPath = packages.map(p => pkg.internal[p].path);
await pkg.loadPyodide('micropip');
await pyodide.globals.basthon.Basthon.micropip_install(packagesPath);
if(callback) {
callback(packages);
}
};
/**
* Loading module (internal or Pyodide).
* Callback function is called twice.
* First on (not already loaded) Pyodide packages,
* then on (not already loaded) internal packages.
*/
pkg.load = async function (packages, callback) {
if( typeof packages === "string" ) {
packages = [packages];
}
if( packages.length === 0 ) {
return;
}
const pyodidePackages = packages.filter(p => pkg.pyodide.has(p));
const internalPackages = packages.filter(p => p in pkg.internal);
await pkg.loadPyodide(pyodidePackages, callback);
await pkg.loadInternal(internalPackages, callback);
};
return pkg;
})();
/**
* Downloading data (bytes array) as filename (opening browser dialog).
......@@ -243,7 +108,6 @@
// get the version of Python from Python
const sys = pyodide.pyimport("sys");
that.pythonVersion = sys.version;
that.packages.init();
return pyodide.loadPackage(that.basthonRoot + "/basthon-py/basthon.js").then(
() => pyodide.runPythonAsync("import basthon"));
};
......@@ -334,20 +198,6 @@
function (event) { callback(event.detail); });
};
/**
* Find modules to import from Python codes.
*/
that.findImports = function (code) {
if( !that.loaded ) { return ; }
let imports = pyodide.globals.basthon.Basthon.find_imports(code);
// manually update internal packages dependencies
for( const i of imports ) {
imports = imports.concat(
(that.packages.internal[i] || {deps: []}).deps);
}
return imports;
};
/**
* Cloning function.
*/
......@@ -355,41 +205,6 @@
// simple trick that is enough for our purpose.
return JSON.parse(JSON.stringify(obj));
};
/**
* Basthon simple code evaluation function (not async).
*/
that.eval = function (code, data=null) {
if( !that.loaded ) { return ; }
return pyodide.globals.basthon.Basthon.eval(code, data);
};
/**
* Load modules required by a piece o code.
*/
that.loadDependencies = async function (code, loadPackageCallback) {
// finding packages, loading, (hacking mpl, turtle, sympy, etc),
const toLoad = that.findImports(code);
return await that.packages.load(toLoad, loadPackageCallback);
};
/**
* Callback function for hacking several modules before import.
*/
that.hackPackagesCallback = function (toLoad) {
const hacked = ["matplotlib", "turtle", "sympy",
"folium", "pandas", "PIL"];
for( const p of toLoad )
if( hacked.includes(p) )
pyodide.globals.basthon.Basthon[`hack_${p}`]();
};
/**
* Load modules required by a piece of code and and hack them.
*/
that.loadDependenciesEvent = async function (code) {
return that.loadDependencies(code, that.hackPackagesCallback);
};
/**
* Basthon async code evaluation function.
......@@ -403,9 +218,8 @@
if( typeof errCallback === 'undefined' ) {
errCallback = function (text) { console.error(text); };
}
// loading dependencies then running
await that.loadDependencies(code, that.hackPackagesCallback);
return await pyodide.globals.basthon.Basthon.shell_eval(code, outCallback, errCallback, data);
// loading dependencies are loaded by eval
return await pyodide.globals.basthon.kernel.eval(code, outCallback, errCallback, data);
};
/**
......@@ -430,53 +244,47 @@
return that.evalAsync(data.code, outCallback, errCallback,
that.hackPackagesCallback, data)
.then(
function (result) {
.then(function (args) {
const result = args[0];
const executionCount = args[1];
let dataEvent = that.clone(data);
dataEvent.execution_count = that.executionCount();
dataEvent.execution_count = executionCount;
if( typeof result !== 'undefined' ) {
dataEvent.result = result;
}
that.dispatchEvent("eval.finished", dataEvent);
},
function (error) {
}, function (args) {
const error = args[0];
const executionCount = args[1];
errCallback(error.toString());
let dataEvent = that.clone(data);
dataEvent.error = error;
dataEvent.execution_count = that.executionCount();
dataEvent.execution_count = executionCount;
that.dispatchEvent("eval.error", dataEvent);
});
};
/**
* Get the current execution count.
*/
that.executionCount = function () {
return pyodide.globals.basthon.Basthon.execution_count;
};
/**
* Restart the kernel.
*/
that.restart = function () {
if( !that.loaded ) { return ; }
return pyodide.globals.basthon.Basthon.restart();
return pyodide.globals.basthon.kernel.restart();
};
/**
* Put a file on the local (emulated) filesystem.
*/
that.putFile = function (filename, content) {
pyodide.globals.Basthon.put_file(filename, content);
pyodide.globals.basthon.kernel.put_file(filename, content);
};
/**
* Put an importable module on the local (emulated) filesystem
* and load dependencies.
*/
that.putModule = async function (filename, content) {
await that.loadDependencies(content, that.hackPackagesCallback);
pyodide.globals.basthon.Basthon.put_module(filename, content);
that.putModule = function (filename, content) {
return pyodide.globals.basthon.kernel.put_module(filename, content);
};
/**
......
"""
This is the Python part of the Basthon Kernel.
"""
import pyodide
import sys
import importlib
import os
import pydoc
from js import document, window
from . import kernel, _hack_builtins
__author__ = "Romain Casati"
......@@ -16,458 +6,12 @@ __license__ = "GNU GPL v3"
__email__ = "romain.casati@basthon.fr"
__all__ = ['Basthon']
class BasthonNamespace(object):
"""
A class that acts as a namespace for Basthon methods.
This is the part of Basthon implemented in Python that would be loaded
in the global namspace. Code evaluation would be executed in separate
namespace that can be cleared (this simulate the kernel restart).
"""
class StreamManager(object):
"""
A class to catch stderr/stdout input during eval.
"""
def __init__(self, stream, flush_callback):
self.stream = stream
self.callback = flush_callback
std = getattr(sys, stream)
self.std = std
self.buff = ""
self.write_bck = std.write
self.flush_bck = std.flush
std.write = self.write
std.flush = self.flush
def __del__(self):
self.close()
def write(self, data):
self.buff += data
# using RE here could improve computation time
if '\n' in data or '\r' in data:
self.flush()
return len(data)
def flush(self):
if not self.buff:
return
self.callback(self.buff)
self.buff = ""
def close(self):
self.flush()
self.std.write = self.write_bck
self.std.flush = self.flush_bck
def __init__(self):
self.execution_count = None
self._namespace = {}
self._user_modules_root = "/basthon_user_modules"
# we don't insert at position 0 since
# https://github.com/iodide-project/pyodide/issues/737#issuecomment-750858417
sys.path.insert(2, self._user_modules_root)
self._init_basthon_internal()
self.start()
def _init_basthon_internal(self):
""" Build the basthon internal object """
basthon = type('basthon', (object,), {})()
basthon.__doc__ = """ This is the global Basthon namespace.
It is used to provide usefull methods for Python in te browser.
"""
def display(obj):
""" Emulating the IPython.core.display.display function """
self.display_event({'display_type': 'multiple',
'content': self.format_repr(obj)})
def display_image(img):
""" Displaying image from numpy array """
from matplotlib import image
import base64
import io
def _repr_png_():
raw = io.BytesIO()
image.imsave(raw, img, format="png", cmap='gray')
raw.seek(0)
return base64.b64encode(raw.read()).decode()
dummy = type('image', (), {})()
dummy._repr_png_ = _repr_png_
display(dummy)
def download(filename):
"""
Download a file from the local filesystem
via a browser dialog.
"""
return self.get_file(filename)
basthon.display = display
basthon.display_image = display_image
basthon.download = download
self.basthon_internal = basthon
def start(self):
"""
Start the Basthon kernel and fill the namespace.
"""
basthon = self.basthon_internal
self.execution_count = 0
self._namespace.clear()
self._namespace.update({
'__name__': '__console__',
'__doc__': None,
'_': '',
'__': '',
'___': '',
'In': [''],
'Out': {},
'display': basthon.display,
'basthon': basthon
})
def stop(self):
"""
Stop the Basthon kernel.
"""
pass
def restart(self):
"""
Restart the Basthon kernel.
"""
self.stop()
self.start()
def roll_in_history(self, code):
""" Manage storing in 'In' ala IPython. """
self._namespace['In'].append(code)
def roll_out_history(self, out):
""" Manage storing in 'Out', _, __, ___ ala IPython. """
outputs = self._namespace['Out']
# out is not always stored
if out is not None and out is not outputs:
outputs[self.execution_count] = out
self._namespace['___'] = self._namespace['__']
self._namespace['__'] = self._namespace['_']
self._namespace['_'] = out
def eval(self, code, data=None):
"""
Kernel function to evaluate Python code.
data can be accessed in code through '__eval_data__' variable
in gobal namespace.
"""
self._namespace['__eval_data__'] = data
res = pyodide.eval_code(code, self._namespace)
# This lines has been commented for p5 compatibility.
# It seems useless to delete since later call will erase it.
# However, if we delete it and there is asynchronous call to
# say eval.display event (like in some p5) we are bad...
# del self._namespace['__eval_data__']
return res
def format_repr(self, obj):
res = {"text/plain": repr(obj)}
if hasattr(obj, "_repr_html_"):
res["text/html"] = obj._repr_html_()
if hasattr(obj, "_repr_svg_"):
res["image/svg+xml"] = obj._repr_svg_()
if hasattr(obj, "_repr_png_"):
res["image/png"] = obj._repr_png_()
return res
def shell_eval(self, code, stdout_callback, stderr_callback, data=None):
"""
Evaluation of Python code with communication managment
with the JS part of Basthon and stdout/stderr catching.
data can be accessed in code through '__eval_data__' variable
in global namespace.
"""
self.execution_count += 1
self.roll_in_history(code)
stdout_manager = BasthonNamespace.StreamManager("stdout", stdout_callback)
stderr_manager = BasthonNamespace.StreamManager("stderr", stderr_callback)
try:
_ = self.eval(code, data=data)
except Exception:
raise
else:
self.roll_out_history(_)
if _ is not None:
return self.format_repr(_)
finally:
stdout_manager.close()
stderr_manager.close()
def find_imports(self, code):
"""
Wrapper around pyodide.find_imports.
"""
if not isinstance(code, str):
try:
code = code.tobytes()
except Exception:
pass
code = code.decode()
return pyodide.find_imports(code)
def micropip_install(self, packages):
""" Load packages using micropip and return a promise. """
import micropip
return micropip.install(packages)
def importables(self):
""" List of all importable modules. """
import sys
import pkgutil
import js
from_sys = set(x for x in sys.modules.keys() if '.' not in x)
from_pkgutil = set(p.name for p in pkgutil.iter_modules())
from_basthon_js = set(x for x in js.window.Basthon.packages.all if '.' not in x)
return sorted(from_sys.union(from_pkgutil, from_basthon_js))
def put_file(self, filepath, content):
"""
Put a file on the (emulated) local filesystem.
"""
dirname, _ = os.path.split(filepath)
if dirname:
os.makedirs(dirname, exist_ok=True)
with open(filepath, 'wb') as f:
f.write(content)
def put_module(self, filename, content):
"""
Put a module (*.py file) on the (emulated) local filesystem
bypassing the Pyodide' single-import-issue by invalidate caches
https://github.com/iodide-project/pyodide/issues/737
/!\ Warning: the dependencies loading is done on the JS side by
basthon.js.
"""
_, filename = os.path.split(filename)
module_path = os.path.join(self._user_modules_root, filename)
self.put_file(module_path, content)
file_finder = sys.path_importer_cache.get(self._user_modules_root)
if file_finder is None:
# can't figure out why this is needed...
importlib.invalidate_caches()
else:
file_finder.invalidate_caches()
def get_file(self, filepath):
"""
Download a file from the (emulated) local filesystem.
"""
_, filename = os.path.split(filepath)
with open(filepath, 'rb') as f:
window.Basthon.download(f.read(), filename)
def display_event(self, data):
""" Dispatching eval.display event with display data. """
display_data = {}
# Updating display data with evaulation data.
# get evaluation data from namespace
eval_data = self._namespace['__eval_data__']
if eval_data is not None:
display_data.update(eval_data)
display_data.update(data)
window.Basthon.dispatchEvent("eval.display", display_data)
def hack_matplotlib(self):
"""
Hack the Wasm backend of matplotlib to render figures.
"""
from matplotlib.backends.wasm_backend import FigureCanvasWasm as wasm_backend
# preserve access to self
this = self
# hacking root node creation
def create_root_element(self):
self.root = document.createElement("div")
return self.root
wasm_backend.create_root_element = create_root_element
# hacking show
if not hasattr(wasm_backend, "_original_show"):
wasm_backend._original_show = wasm_backend.show
def show(self):
res = self._original_show()
this.display_event({"display_type": "matplotlib",
"content": self.root})
return res
show.__doc__ = wasm_backend._original_show.__doc__
wasm_backend.show = show
def hack_turtle(self):
"""
Hack Turtle to render figures.