Commit e37f9494 authored by Damien Wojtowicz's avatar Damien Wojtowicz
Browse files

Dynamic configuration for providers + compilation fixes + renames to Nebula

parent 3c2d7aa7
......@@ -629,7 +629,7 @@ to attach them to the start of each source file to most effectively
state the exclusion of warranty; and each file should have at least
the "copyright" line and a pointer to where the full notice is found.
Odyr
Nebula
Copyright (C) 2020 Damien Wojtowicz
This program is free software: you can redistribute it and/or modify
......
# Nebula
Middleware enabling the formulation and execution of queries in a multi-cloud DBaaS environment.
Middleware enabling the formulation and execution of queries in a multi-cloud DBaaS environment presented in the
following paper.
D. T. Wojtowicz, S. Yin, F. Morvan and A. Hameurlain, "Cost-Effective Dynamic Optimisation for Multi-Cloud Queries," _
2021 IEEE 14th International Conference on Cloud Computing (CLOUD)_, 2021, pp. 387-397
[Get the PDF here !](https://hal-univ-tlse3.archives-ouvertes.fr/hal-03428073)
......@@ -13,16 +13,16 @@ import requests
#
########################################################################################################################
parser = argparse.ArgumentParser(description="Runs benchmarks for Odyr.")
parser = argparse.ArgumentParser(description="Runs benchmarks for Nebula.")
parser.add_argument("queries_dir", type=str, help="Directory containing SQL queries files")
parser.add_argument("-o", "--out_path", type=str,
default=f"results_{str(datetime.datetime.now()).replace(' ', '_').replace(':', '-')}",
help="Output file path.")
parser.add_argument("-n", "--nb_round", default=10, type=int, help="Number of rounds of queries to execute")
parser.add_argument("-u", "--url", default="localhost:4999", type=str, help="Odyr url")
parser.add_argument("-u", "--url", default="localhost:4999", type=str, help="Nebula url")
args = parser.parse_args()
ODYR_URL = args.url
NEBULA_URL = args.url
QUERIES_DIR = args.queries_dir
OUT_PATH = args.out_path
OUT_PATH_SLA = f"{args.out_path}/sla.csv"
......@@ -111,11 +111,11 @@ class Query:
def __generate_sla(self, qname):
"""Generates a SLA for this specific query"""
global ODYR_URL
global NEBULA_URL
start = datetime.datetime.now()
message = requests.post(
f"http://{ODYR_URL}/estimate",
f"http://{NEBULA_URL}/estimate",
data={"query": self.sql, "query_name": qname}
)
message = json.loads(message.content)
......@@ -162,11 +162,11 @@ class Query:
def __execute(self, name, mode):
"""Executes the query. Mode is either 'static' or 'dynamic'."""
global ODYR_URL
global NEBULA_URL
start = datetime.datetime.now()
message = requests.post(
f"http://{ODYR_URL}/execute",
f"http://{NEBULA_URL}/execute",
data={"query": self.sql, "query_name": name, "method": mode}
)
message = json.loads(message.content)
......
......@@ -13,7 +13,7 @@ import requests
#
########################################################################################################################
parser = argparse.ArgumentParser(description="Runs benchmarks for Odyr.")
parser = argparse.ArgumentParser(description="Runs benchmarks for Nebula.")
parser.add_argument("queries_dir", type=str, help="Directory containing SQL queries files")
parser.add_argument("-o", "--out_path", type=str,
default=f"results_{str(datetime.datetime.now()).replace(' ', '_').replace(':', '-')}",
......@@ -21,10 +21,10 @@ parser.add_argument("-o", "--out_path", type=str,
parser.add_argument("-n", "--nb_round", default=10, type=int, help="Number of rounds of queries to execute")
parser.add_argument("-u", "--url", default="localhost:4999", type=str, help="Odyr url")
parser.add_argument("-m", "--mode", default="dynamic", type=str, choices=["static", "dynamic"],
help="Execution mode for Odyr")
help="Execution mode for Nebula")
args = parser.parse_args()
ODYR_URL = args.url
NEBULA_URL = args.url
QUERIES_DIR = args.queries_dir
OUT_PATH = args.out_path
# OUT_PATH_SLA = f"{args.out_path}/sla.csv"
......@@ -42,7 +42,7 @@ MODE = args.mode
def quotes_request(q_name, sql):
message = requests.post(
f"http://{ODYR_URL}/estimate",
f"http://{NEBULA_URL}/estimate",
data={"query": sql, "query_name": q_name}
)
return json.loads(message.content)
......@@ -52,7 +52,7 @@ def execute_query(q_name, sql, quote, mode=MODE):
print({"query_name": q_name, "method": mode, "quotation_name": quote})
start = datetime.datetime.now()
message = requests.post(
f"http://{ODYR_URL}/execute",
f"http://{NEBULA_URL}/execute",
data={"query": sql, "query_name": q_name, "method": mode, "quotation_name": quote}
)
message = json.loads(message.content)
......
......@@ -3,14 +3,14 @@ from typing import List, Tuple
from sly import Parser
from sly.lex import LexError
from odyr.compiler.exceptions import SyntaxError
from odyr.compiler.lexer import SqlLexer
from odyr.compiler.tmp_operators import TmpAttribute, TmpProjectionStar, TmpRelation
from odyr.database.metabase import Relation, Provider, Attribute, Session
from odyr.query.operators import Ensembly, ReadTable
from nebula.compiler.exceptions import SyntaxError
from nebula.compiler.lexer import SqlLexer
from nebula.compiler.tmp_operators import TmpAttribute, TmpProjectionStar, TmpRelation
from nebula.database.metabase import Relation, Provider, Attribute, Session
from nebula.query.operators import Ensembly, ReadTable
class SqlParser(Parser) :
class SqlParser(Parser):
"""Parser for a SQL Data Interrogation Languge grammar, returning an execution plan.
Implementation follows the tutorial at https://sly.readthedocs.io/en/latest/sly.html#writing-a-parser
......
......@@ -9,7 +9,7 @@ from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship, sessionmaker
# Getting db file and cleaning it up
from odyr.query.sla import SLA
from nebula.query.sla import SLA
dbname = str(sys.argv[0]).split("/")[-1].split(".py")[0]
try:
......@@ -98,12 +98,16 @@ class Provider(Base) :
else:
self.__drop_entity(name)
def admin(self, query, views_info=None, no_update=False):
def admin(self, query, views_info=None, no_update=False, debug=False):
data = {"query": query}
if views_info is not None:
data["views_info"] = views_info
if debug:
print("== SENDING MESSAGE TO ", f"http://{self.url}:{self.port}/admin")
print(data)
message = requests.post(
f"http://{self.url}:{self.port}/admin",
data=data
......
......@@ -2,8 +2,8 @@ import copy
import threading
from threading import Thread
from odyr.database.metabase import Provider
from odyr.query.query import Query
from nebula.database.metabase import Provider
from nebula.query.query import Query
class Bill:
......@@ -163,7 +163,7 @@ class TaskWorker(Thread):
# Creates a table based on the foreign table
query = f"CREATE UNLOGGED TABLE {self.subquery_name} AS SELECT * FROM {self.subquery_name}_tmp ;"
self.p_destination.admin(query, no_update=True)
self.p_destination.admin(query, no_update=True, debug=True)
# Cleaning in another thread, in order to make it in parallel
th = threading.Thread(target=self.__clean_after_transfer,
......
......@@ -4,8 +4,8 @@ from typing import Tuple, List
import networkx as nx
from odyr.database.metabase import Provider
from odyr.engine.base import ExecutionEngine, TaskWorker
from nebula.database.metabase import Provider
from nebula.engine.base import ExecutionEngine, TaskWorker
class DynamicEngine(ExecutionEngine):
......@@ -181,8 +181,8 @@ class DynamicTaskWorker(TaskWorker):
next_n = list(potential_nexts)[0]
# Check for first-choice eligibility
is_first_choice = time <= 1.1 * self.engine.target_m \
and cost <= 1.1 * self.engine.target_c
is_first_choice = time <= 1.1 * self.engine.target_d \
and cost <= 1.1 * self.engine.target_m
return {
"cost": cost,
"time": time,
......
......@@ -3,15 +3,15 @@ from typing import List, Tuple
import networkx as nx
from odyr.database.metabase import Provider
from odyr.engine.base import ExecutionEngine, Bill, TaskWorker
from nebula.database.metabase import Provider
from nebula.engine.base import ExecutionEngine, Bill, TaskWorker
class StaticTaskWorker(TaskWorker):
"""Abstract class for a task worker"""
def __init__(self, engine, p_origin: Provider, subquery: str, subquery_name: str, p_destination: Provider,
end_name=None):
end_name=None):
super().__init__(engine, p_origin, subquery, subquery_name, p_destination)
self.subquery_name = subquery_name
self.end_name = end_name if end_name is not None else self.subquery_name
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment