diff --git a/backend/app/apps.py b/backend/app/apps.py index ed327d22f..bcfe39bb2 100644 --- a/backend/app/apps.py +++ b/backend/app/apps.py @@ -2,5 +2,5 @@ class AppConfig(AppConfig): - default_auto_field = 'django.db.models.BigAutoField' - name = 'app' + default_auto_field = "django.db.models.BigAutoField" + name = "app" diff --git a/backend/app/urls.py b/backend/app/urls.py index 78eadae57..f3b24810f 100644 --- a/backend/app/urls.py +++ b/backend/app/urls.py @@ -2,8 +2,8 @@ from . import views urlpatterns = [ - path('build', views.build, name='build'), - path('download', views.download, name='download'), - path('install_block', views.install_block, name='install_block'), - path('installed_blocks', views.installed_blocks, name='installed_blocks'), + path("build", views.build, name="build"), + path("download", views.download, name="download"), + path("install_block", views.install_block, name="install_block"), + path("installed_blocks", views.installed_blocks, name="installed_blocks"), ] diff --git a/backend/app/views.py b/backend/app/views.py index 9e6ac6d74..829173bbf 100644 --- a/backend/app/views.py +++ b/backend/app/views.py @@ -9,6 +9,7 @@ import os from django.conf import settings + @csrf_exempt def build(request): try: @@ -19,48 +20,52 @@ def build(request): return HttpResponseBadRequest(e.msg) - def download(request): - return JsonResponse({'status': 'request received'}) + return JsonResponse({"status": "request received"}) @csrf_exempt def install_block(request): try: data = json.loads(request.body) - block_name = data.get('package', {}).get('name') or data.get('name', 'Untitled') - - custom_blocks_dir = os.path.join(settings.BASE_DIR, 'custom_blocks') + block_name = data.get("package", {}).get("name") or data.get("name", "Untitled") + + custom_blocks_dir = os.path.join(settings.BASE_DIR, "custom_blocks") if not os.path.exists(custom_blocks_dir): os.makedirs(custom_blocks_dir) - - safe_filename = "".join([c for c in block_name if c.isalpha() or c.isdigit() or c in (' ')]).rstrip() + + safe_filename = "".join( + [c for c in block_name if c.isalpha() or c.isdigit() or c in (" ")] + ).rstrip() filepath = os.path.join(custom_blocks_dir, f"{safe_filename}.vc3") - - with open(filepath, 'w') as f: + + with open(filepath, "w") as f: json.dump(data, f, indent=4) - - return JsonResponse({'status': 'success', 'saved_as': os.path.basename(filepath)}) + + return JsonResponse( + {"status": "success", "saved_as": os.path.basename(filepath)} + ) except Exception as e: return HttpResponseBadRequest(str(e)) + def installed_blocks(request): try: - custom_blocks_dir = os.path.join(settings.BASE_DIR, 'custom_blocks') + custom_blocks_dir = os.path.join(settings.BASE_DIR, "custom_blocks") if not os.path.exists(custom_blocks_dir): - return JsonResponse({'blocks': []}) - + return JsonResponse({"blocks": []}) + blocks = [] for filename in os.listdir(custom_blocks_dir): - if filename.endswith('.vc3'): + if filename.endswith(".vc3"): filepath = os.path.join(custom_blocks_dir, filename) try: - with open(filepath, 'r') as f: + with open(filepath, "r") as f: block_data = json.load(f) blocks.append(block_data) except Exception as e: print(f"Failed to load block {filename}: {e}") - - return JsonResponse({'blocks': blocks}) + + return JsonResponse({"blocks": blocks}) except Exception as e: return HttpResponseBadRequest(str(e)) diff --git a/backend/backend/asgi.py b/backend/backend/asgi.py index 14a707ee6..0bdddee5e 100644 --- a/backend/backend/asgi.py +++ b/backend/backend/asgi.py @@ -11,6 +11,6 @@ from django.core.asgi import get_asgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") application = get_asgi_application() diff --git a/backend/backend/settings.py b/backend/backend/settings.py index 162c5a94f..114a6cbcd 100644 --- a/backend/backend/settings.py +++ b/backend/backend/settings.py @@ -18,7 +18,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent env = environ.Env() -environ.Env.read_env(str(BASE_DIR / '.env')) +environ.Env.read_env(str(BASE_DIR / ".env")) # Change this to your desired port number DESIRED_PORT = 8080 @@ -26,12 +26,12 @@ # See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = env.str('DJANGO_SECRET_KEY') +SECRET_KEY = env.str("DJANGO_SECRET_KEY") # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = env.bool('DJANGO_DEBUG', default=False) +DEBUG = env.bool("DJANGO_DEBUG", default=False) -ALLOWED_HOSTS = env.list('DJANGO_ALLOWED_HOSTS', default=['http://localhost:80']) +ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["http://localhost:80"]) # ALLOWED_HOSTS = [ # 'localhost', # '127.0.0.1', @@ -43,56 +43,55 @@ # Application definition INSTALLED_APPS = [ - 'django.contrib.admin', - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'django.contrib.messages', - 'django.contrib.staticfiles', - - 'corsheaders', - 'app', + "django.contrib.admin", + "django.contrib.auth", + "django.contrib.contenttypes", + "django.contrib.sessions", + "django.contrib.messages", + "django.contrib.staticfiles", + "corsheaders", + "app", ] MIDDLEWARE = [ - 'corsheaders.middleware.CorsMiddleware', - 'django.middleware.security.SecurityMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.common.CommonMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.messages.middleware.MessageMiddleware', - 'django.middleware.clickjacking.XFrameOptionsMiddleware', + "corsheaders.middleware.CorsMiddleware", + "django.middleware.security.SecurityMiddleware", + "django.contrib.sessions.middleware.SessionMiddleware", + "django.middleware.common.CommonMiddleware", + "django.middleware.csrf.CsrfViewMiddleware", + "django.contrib.auth.middleware.AuthenticationMiddleware", + "django.contrib.messages.middleware.MessageMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", ] -ROOT_URLCONF = 'backend.urls' +ROOT_URLCONF = "backend.urls" TEMPLATES = [ { - 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [], - 'APP_DIRS': True, - 'OPTIONS': { - 'context_processors': [ - 'django.template.context_processors.debug', - 'django.template.context_processors.request', - 'django.contrib.auth.context_processors.auth', - 'django.contrib.messages.context_processors.messages', + "BACKEND": "django.template.backends.django.DjangoTemplates", + "DIRS": [], + "APP_DIRS": True, + "OPTIONS": { + "context_processors": [ + "django.template.context_processors.debug", + "django.template.context_processors.request", + "django.contrib.auth.context_processors.auth", + "django.contrib.messages.context_processors.messages", ], }, }, ] -WSGI_APPLICATION = 'backend.wsgi.application' +WSGI_APPLICATION = "backend.wsgi.application" # Database # https://docs.djangoproject.com/en/3.2/ref/settings/#databases DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - 'NAME': BASE_DIR / 'db.sqlite3', + "default": { + "ENGINE": "django.db.backends.sqlite3", + "NAME": BASE_DIR / "db.sqlite3", } } @@ -102,16 +101,16 @@ AUTH_PASSWORD_VALIDATORS = [ { - 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", }, { - 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", }, ] @@ -119,9 +118,9 @@ # Internationalization # https://docs.djangoproject.com/en/3.2/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = "en-us" -TIME_ZONE = 'UTC' +TIME_ZONE = "UTC" USE_I18N = True @@ -133,33 +132,33 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ -STATIC_URL = '/static/' +STATIC_URL = "/static/" STATICFILES_DIRS = [ BASE_DIR / "staticfiles", ] -STATIC_ROOT = 'static' +STATIC_ROOT = "static" # Default primary key field type # https://docs.djangoproject.com/en/3.2/ref/settings/#default-auto-field -DEFAULT_AUTO_FIELD = 'django.db.models.BigAutoField' +DEFAULT_AUTO_FIELD = "django.db.models.BigAutoField" if DEBUG: # Accept a comma-separated list of frontend hosts (e.g. http://localhost:4000) # Also automatically append standard fallback ports if the primary port gets busy - base_origins = [origin.strip() for origin in env.str('VISUAL_CIRCUIT_FRONTEND_HOST').split(',')] + base_origins = [ + origin.strip() for origin in env.str("VISUAL_CIRCUIT_FRONTEND_HOST").split(",") + ] CORS_ALLOWED_ORIGINS = list(base_origins) - + # If using localhost:4000, dynamically add 4001 and 4002 as fallbacks for origin in base_origins: - if origin.endswith(':4000'): - CORS_ALLOWED_ORIGINS.extend([ - origin.replace(':4000', ':4001'), - origin.replace(':4000', ':4002') - ]) - + if origin.endswith(":4000"): + CORS_ALLOWED_ORIGINS.extend( + [origin.replace(":4000", ":4001"), origin.replace(":4000", ":4002")] + ) # Allow frontend to read the Content-Disposition header to get the correct .zip filename - CORS_EXPOSE_HEADERS = ['Content-Disposition'] + CORS_EXPOSE_HEADERS = ["Content-Disposition"] diff --git a/backend/backend/urls.py b/backend/backend/urls.py index ae782ba32..803191ade 100644 --- a/backend/backend/urls.py +++ b/backend/backend/urls.py @@ -2,6 +2,6 @@ from django.urls import include, path urlpatterns = [ - path('admin/', admin.site.urls), - path('api/', include('app.urls')), + path("admin/", admin.site.urls), + path("api/", include("app.urls")), ] diff --git a/backend/backend/wsgi.py b/backend/backend/wsgi.py index e40ee1efa..5a123d6f9 100644 --- a/backend/backend/wsgi.py +++ b/backend/backend/wsgi.py @@ -11,6 +11,6 @@ from django.core.wsgi import get_wsgi_application -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") application = get_wsgi_application() diff --git a/backend/manage.py b/backend/manage.py index eb6431e2c..ae97db8ba 100644 --- a/backend/manage.py +++ b/backend/manage.py @@ -1,12 +1,13 @@ #!/usr/bin/env python """Django's command-line utility for administrative tasks.""" + import os import sys def main(): """Run administrative tasks.""" - os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'backend.settings') + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "backend.settings") try: from django.core.management import execute_from_command_line except ImportError as exc: @@ -18,5 +19,5 @@ def main(): execute_from_command_line(sys.argv) -if __name__ == '__main__': +if __name__ == "__main__": main() diff --git a/backend/staticfiles/synthesis/lib/exceptions.py b/backend/staticfiles/synthesis/lib/exceptions.py index 37ba359fa..06e10959c 100644 --- a/backend/staticfiles/synthesis/lib/exceptions.py +++ b/backend/staticfiles/synthesis/lib/exceptions.py @@ -5,5 +5,6 @@ class InvalidOutputNameException(Exception): class InvalidInputNameException(Exception): """Raised when Input name has not been declared in ports""" + class InvalidParameterNameException(Exception): """Raised when Parameter name has not been declared in ports""" diff --git a/backend/staticfiles/synthesis/lib/inputs.py b/backend/staticfiles/synthesis/lib/inputs.py index 3cfcde088..b8bc10f04 100644 --- a/backend/staticfiles/synthesis/lib/inputs.py +++ b/backend/staticfiles/synthesis/lib/inputs.py @@ -12,6 +12,7 @@ def create_readonly_wire(name): shm = None return shm + def create_number_wire(name, size): try: shm = shared_memory.SharedMemory(name=name) @@ -19,13 +20,18 @@ def create_number_wire(name, size): shm = shared_memory.SharedMemory(name=name, create=True, size=size) return shm + class Inputs: ENABLE_NAME = "Enable" def __init__(self, input_data) -> None: self.inputs = input_data - self._enable_data = self.inputs[Inputs.ENABLE_NAME] if Inputs.ENABLE_NAME in self.inputs else None + self._enable_data = ( + self.inputs[Inputs.ENABLE_NAME] + if Inputs.ENABLE_NAME in self.inputs + else None + ) def read(self, name): if self.inputs.get(name) is None: @@ -36,7 +42,7 @@ def read(self, name): # Read data from the different buffers of the SHM Objects dim = create_ndbuffer((1,), np.int64, self.inputs[name]["dim"].buf)[:][0] shape = create_ndbuffer((dim,), np.int64, self.inputs[name]["shape"].buf) - type = create_ndbuffer((1,), ' bool: # No enable wire used, so enabled by default @@ -165,16 +175,14 @@ def enabled(self) -> bool: return np.isclose(_enabled, np.array([1.0])) - - @enabled.setter def enabled(self, _enabled: bool): # If no wire exists, we cannot set anything, it is true by default # TODO: Ideally one should be able to trigger a block on and off even without an enable wire - # Can we force there to be an enable slot in all blocks? Or is another approach a better idea? + # Can we force there to be an enable slot in all blocks? Or is another approach a better idea? if self._enable_data is None: return - + self._enable_data["lock"].acquire() if self._enable_data.get("created", False): wire_val = np.array([1]) @@ -185,10 +193,13 @@ def enabled(self, _enabled: bool): # print("Disabling wire") wire_val = np.array([0]) - wire_data = np.ndarray(wire_val.shape, dtype=wire_val.dtype, buffer=self._enable_data["data"].buf) + wire_data = np.ndarray( + wire_val.shape, + dtype=wire_val.dtype, + buffer=self._enable_data["data"].buf, + ) wire_data[:] = wire_val[:] - else: # Wire doesn't exist yet: this implementation requires it to be # created and set here. @@ -198,11 +209,16 @@ def enabled(self, _enabled: bool): # Value of the wire to be set wire_val = np.array([1]) # Create a new shared memory object in the "data" key of the _enable_data dictionary - self._enable_data["data"] = create_number_wire(wire_name, wire_val.nbytes) - data_wire = np.ndarray(wire_val.shape, dtype=np.float64, buffer=self._enable_data["data"].buf) + self._enable_data["data"] = create_number_wire( + wire_name, wire_val.nbytes + ) + data_wire = np.ndarray( + wire_val.shape, + dtype=np.float64, + buffer=self._enable_data["data"].buf, + ) data_wire[:] = wire_val[:] # Mark wire as created, since it has been created self._enable_data["created"] = True self._enable_data["lock"].release() - \ No newline at end of file diff --git a/backend/staticfiles/synthesis/lib/outputs.py b/backend/staticfiles/synthesis/lib/outputs.py index 7b8cd448a..60784ccff 100644 --- a/backend/staticfiles/synthesis/lib/outputs.py +++ b/backend/staticfiles/synthesis/lib/outputs.py @@ -23,17 +23,17 @@ def check_type(self, typestr): # Isolates numbers at the end of the data type chars = int(typestr[2:]) # In case of float and integer data types, default to 64 bit - if typestr[1] == 'i' or typestr[1] == 'f': + if typestr[1] == "i" or typestr[1] == "f": suffix_no = chars if chars > 8 else 8 # In case of Unicode String (U) or String (S) default to 64 chars - elif typestr[1] == 'U' or typestr[1] == 'S': + elif typestr[1] == "U" or typestr[1] == "S": suffix_no = chars if chars > 64 else 64 # Otherwise simply let the original number be else: suffix_no = chars # Combine with the type with the appropriate suffix final_type = typestr[:2] + str(suffix_no) - return (final_type) + return final_type def share(self, name, data): if self.outputs.get(name) is None: @@ -48,8 +48,8 @@ def share(self, name, data): dim = np.array([len(shape)]) # Check if the data type needs modifications, get the modified type after calling the function final_type = self.check_type(data.dtype.str) - type = np.array([final_type], dtype=' 256 else 256 + data_size = data.nbytes if data.nbytes > 256 else 256 data_wire = self._create_wire(self.outputs[name]["wire"], data_size) # Create array that accesses SHM Object's buffer to store the dimensions of the data being passed self.outputs[name]["dim"] = create_ndbuffer((1,), np.int64, dim_wire.buf) self.outputs[name]["dim"][:] = dim[:] # Create array that accesses SHM Object's buffer to store the type of the data being passed - self.outputs[name]["type"] = create_ndbuffer((1,), ' None: def read_number(self, name): if self.parameters.get(name) is None: raise InvalidParameterNameException(f"{name} is not declared in parameters") - + return float(self.parameters[name]) def read_string(self, name): if self.parameters.get(name) is None: raise InvalidParameterNameException(f"{name} is not declared in parameters") - + return str(self.parameters[name]) def read_bool(self, name): if self.parameters.get(name) is None: raise InvalidParameterNameException(f"{name} is not declared in parameters") - return bool(self.parameters[name]) \ No newline at end of file + return bool(self.parameters[name]) diff --git a/backend/staticfiles/synthesis/lib/utils.py b/backend/staticfiles/synthesis/lib/utils.py index 7adf42c27..428b7422b 100644 --- a/backend/staticfiles/synthesis/lib/utils.py +++ b/backend/staticfiles/synthesis/lib/utils.py @@ -1,6 +1,7 @@ import numpy as np from time import sleep, time + def create_ndbuffer(shape, dtype, buffer): return np.ndarray(shape, dtype=dtype, buffer=buffer) diff --git a/backend/staticfiles/synthesis/main.py b/backend/staticfiles/synthesis/main.py index 6bcca9bfb..c15d31305 100644 --- a/backend/staticfiles/synthesis/main.py +++ b/backend/staticfiles/synthesis/main.py @@ -14,8 +14,8 @@ from lib.parameters import Parameters from lib.utils import Synchronise -BLOCK_DIRECTORY = 'modules' -FUNCTION_NAME = 'main' +BLOCK_DIRECTORY = "modules" +FUNCTION_NAME = "main" def clean_shared_memory(signum, frame, names, processes): @@ -28,7 +28,7 @@ def clean_shared_memory(signum, frame, names, processes): all_names.extend([name + "_dim" for name in names]) all_names.extend([name + "_shape" for name in names]) all_names.extend([name + "_type" for name in names]) - + # Clean all shared memory for name in all_names: try: @@ -85,9 +85,13 @@ def main(): ) if source["name"] in block_data[source["block"]]["outputs"]: - wire_name = block_data[source["block"]]["outputs"][source["name"]]["wire"] + wire_name = block_data[source["block"]]["outputs"][source["name"]][ + "wire" + ] elif target["name"] in block_data[target["block"]]["inputs"]: - wire_name = block_data[target["block"]]["inputs"][target["name"]]["wire"] + wire_name = block_data[target["block"]]["inputs"][target["name"]][ + "wire" + ] else: wire_name = "".join( random.choices(string.ascii_uppercase + string.digits, k=10) @@ -96,21 +100,33 @@ def main(): # If a new wire, add it to dictionary and also keep track of its lock if wire_name not in all_wires: all_wires[wire_name] = Lock() - output_data = {source["name"]: {"wire": wire_name, "lock": all_wires[wire_name]}} - input_data = {target["name"]: {"wire": wire_name, "lock": all_wires[wire_name]}} + output_data = { + source["name"]: {"wire": wire_name, "lock": all_wires[wire_name]} + } + input_data = { + target["name"]: {"wire": wire_name, "lock": all_wires[wire_name]} + } block_data[source["block"]]["outputs"].update(output_data) block_data[target["block"]]["inputs"].update(input_data) - for block in blocks: if blocks[block]["type"] in parameters: - block_data[block] = block_data.get(block, {"inputs": {}, "outputs": {}, "parameters": {}}) - for param in parameters[ blocks[block]["type"]]: + block_data[block] = block_data.get( + block, {"inputs": {}, "outputs": {}, "parameters": {}} + ) + for param in parameters[blocks[block]["type"]]: parameter_data = {param["name"]: param["value"]} block_data[block]["parameters"].update(parameter_data) - if block in synchronize_frequency or blocks[block]["type"] in synchronize_frequency: - block_data[block] = block_data.get(block, {"inputs": {}, "outputs": {}, "parameters": {}}) - block_data[block]["frequency"] = synchronize_frequency.get(block, synchronize_frequency.get(blocks[block]["type"], 30)) + if ( + block in synchronize_frequency + or blocks[block]["type"] in synchronize_frequency + ): + block_data[block] = block_data.get( + block, {"inputs": {}, "outputs": {}, "parameters": {}} + ) + block_data[block]["frequency"] = synchronize_frequency.get( + block, synchronize_frequency.get(blocks[block]["type"], 30) + ) processes = [] @@ -132,12 +148,19 @@ def main(): processes.append( multiprocessing.Process( target=method, - args=(inputs, outputs, parameters, Synchronise(1 / (freq if freq != 0 else 30))) + args=( + inputs, + outputs, + parameters, + Synchronise(1 / (freq if freq != 0 else 30)), + ), ) ) # Register handler for Ctrl+C - param_func = functools.partial(clean_shared_memory, names=all_wires, processes=processes) + param_func = functools.partial( + clean_shared_memory, names=all_wires, processes=processes + ) signal.signal(signal.SIGINT, param_func) for process in processes: diff --git a/backend/staticfiles/synthesis/utils/__init__.py b/backend/staticfiles/synthesis/utils/__init__.py index 139597f9c..8b1378917 100644 --- a/backend/staticfiles/synthesis/utils/__init__.py +++ b/backend/staticfiles/synthesis/utils/__init__.py @@ -1,2 +1 @@ - diff --git a/backend/staticfiles/synthesis/utils/models/__init__.py b/backend/staticfiles/synthesis/utils/models/__init__.py index 139597f9c..8b1378917 100644 --- a/backend/staticfiles/synthesis/utils/models/__init__.py +++ b/backend/staticfiles/synthesis/utils/models/__init__.py @@ -1,2 +1 @@ - diff --git a/backend/synthesis/file_utils.py b/backend/synthesis/file_utils.py index 4e79117cf..a5af367d7 100644 --- a/backend/synthesis/file_utils.py +++ b/backend/synthesis/file_utils.py @@ -1,16 +1,17 @@ import zipfile from io import BytesIO -# Thanks to stack overflow :) + +# Thanks to stack overflow :) # https://stackoverflow.com/questions/2463770/python-in-memory-zip-library class InMemoryZip(object): def __init__(self): # Create the in-memory file-like object self.zip = BytesIO() - def append(self, filename_in_zip : str, file_contents: str): - '''Appends a file with name filename_in_zip and contents of - file_contents to the in-memory zip.''' + def append(self, filename_in_zip: str, file_contents: str): + """Appends a file with name filename_in_zip and contents of + file_contents to the in-memory zip.""" # Get a handle to the in-memory zip in append mode zf = zipfile.ZipFile(self.zip, "a", zipfile.ZIP_DEFLATED, False) @@ -24,4 +25,4 @@ def append(self, filename_in_zip : str, file_contents: str): def get_zip(self) -> BytesIO: self.zip.seek(0) - return self.zip \ No newline at end of file + return self.zip diff --git a/backend/synthesis/synthesis.py b/backend/synthesis/synthesis.py index fa3dc1966..9b8437f3c 100644 --- a/backend/synthesis/synthesis.py +++ b/backend/synthesis/synthesis.py @@ -8,38 +8,39 @@ import ast import sys -BLOCK_DIRECTORY = 'modules' +BLOCK_DIRECTORY = "modules" OPTIONAL_FILES = { - 'FaceDetector' : 'utils/models/haar_cascade/**/*', - 'ObjectDetector': 'utils/models/yolov3/**/*' + "FaceDetector": "utils/models/haar_cascade/**/*", + "ObjectDetector": "utils/models/yolov3/**/*", } BLOCK_DEPENDENCIES = { - 'FaceDetector': ['opencv-python', 'numpy'], - 'ObjectDetector': ['opencv-python', 'numpy'], - 'ContourDetector': ['opencv-python', 'numpy'], - 'Cropper': ['opencv-python', 'numpy'], - 'ColorFilter': ['opencv-python', 'numpy'], - 'Blur': ['opencv-python', 'numpy'], - 'Dilation': ['opencv-python', 'numpy'], - 'EdgeDetector': ['opencv-python', 'numpy'], - 'Erosion': ['opencv-python', 'numpy'], - 'Threshold': ['opencv-python', 'numpy'], - 'VideoStreamer': ['opencv-python', 'numpy'], - 'ImageRead': ['opencv-python', 'numpy'] + "FaceDetector": ["opencv-python", "numpy"], + "ObjectDetector": ["opencv-python", "numpy"], + "ContourDetector": ["opencv-python", "numpy"], + "Cropper": ["opencv-python", "numpy"], + "ColorFilter": ["opencv-python", "numpy"], + "Blur": ["opencv-python", "numpy"], + "Dilation": ["opencv-python", "numpy"], + "EdgeDetector": ["opencv-python", "numpy"], + "Erosion": ["opencv-python", "numpy"], + "Threshold": ["opencv-python", "numpy"], + "VideoStreamer": ["opencv-python", "numpy"], + "ImageRead": ["opencv-python", "numpy"], } -PROJECT_FILE_EXTENSION = '.vc3' +PROJECT_FILE_EXTENSION = ".vc3" COMMON_PIP_ALIASES = { - 'cv2': 'opencv-python', - 'sklearn': 'scikit-learn', - 'PIL': 'Pillow', - 'bs4': 'beautifulsoup4', - 'yaml': 'pyyaml' + "cv2": "opencv-python", + "sklearn": "scikit-learn", + "PIL": "Pillow", + "bs4": "beautifulsoup4", + "yaml": "pyyaml", } + def get_number_or_default(num, default): try: num = float(num) @@ -47,45 +48,70 @@ def get_number_or_default(num, default): except ValueError: return default -def syntheize_modules(data: dict, zipfile: InMemoryZip) -> Tuple[InMemoryZip, Dict[str, bool]]: - '''Synthesize python code for different blocks as well as user code blocks. + +def syntheize_modules( + data: dict, zipfile: InMemoryZip +) -> Tuple[InMemoryZip, Dict[str, bool]]: + """Synthesize python code for different blocks as well as user code blocks. Different blocks present in the project are collected. Parameters of each dependency block as well as constant blocks are collected. - Blocks, parameters and the connections (wires) between the blocks stored in a + Blocks, parameters and the connections (wires) between the blocks stored in a JSON file. - ''' + """ dependencies = {} blocks = {} parameters = {} synhronize_frequency = {} optional_files = {} project_dependencies = set() - wire_comp = data['design']['graph']['wires'] # Retrieve all wire connections from the design graph + wire_comp = data["design"]["graph"][ + "wires" + ] # Retrieve all wire connections from the design graph dep_no = {} # Dictionary to store the number of blocks of each type # Function to process dependencies and extract block information - def process_dependency(dep, zipfile, synhronize_frequency, optional_files, dep_no, parameters, dependencies): + def process_dependency( + dep, + zipfile, + synhronize_frequency, + optional_files, + dep_no, + parameters, + dependencies, + ): # Iterate over each dependency for key, dependency in dep.items(): - - components = dependency['design']['graph']['blocks'] # Retrieve all blocks in the dependency - wire_comp.extend(dependency['design']['graph']['wires']) # Add all wires from the dependency to the main wire list + + components = dependency["design"]["graph"][ + "blocks" + ] # Retrieve all blocks in the dependency + wire_comp.extend( + dependency["design"]["graph"]["wires"] + ) # Add all wires from the dependency to the main wire list # Iterate over each block in the dependency for block in components: - block_id = block['id'] - block_type = block['type'] + block_id = block["id"] + block_type = block["type"] # Check if the block is of type 'basic.code' - if block_type == 'basic.code': - - script = block['data']['code'] # Retrieve the script code from the block's data - script_name = dependency['package']['name'] # Retrieve the script name from the dependency package - synhronize_frequency[block_id] = get_number_or_default(block['data'].get('frequency', 30), 30) # Set the synchronization frequency, defaulting to 30 - + if block_type == "basic.code": + + script = block["data"][ + "code" + ] # Retrieve the script code from the block's data + script_name = dependency["package"][ + "name" + ] # Retrieve the script name from the dependency package + synhronize_frequency[block_id] = get_number_or_default( + block["data"].get("frequency", 30), 30 + ) # Set the synchronization frequency, defaulting to 30 + # Check if the script name is already in optional files if script_name in optional_files: - optional_files[script_name] = True # Mark the script as required in optional files + optional_files[script_name] = ( + True # Mark the script as required in optional files + ) if script_name in BLOCK_DEPENDENCIES: project_dependencies.update(BLOCK_DEPENDENCIES[script_name]) @@ -93,93 +119,137 @@ def process_dependency(dep, zipfile, synhronize_frequency, optional_files, dep_n # AST automated dependency parsing try: tree = ast.parse(script) - stdlib_names = getattr(sys, 'stdlib_module_names', set()) + stdlib_names = getattr(sys, "stdlib_module_names", set()) for node in ast.walk(tree): module_name = None if isinstance(node, ast.Import): for alias in node.names: - module_name = alias.name.split('.')[0] + module_name = alias.name.split(".")[0] elif isinstance(node, ast.ImportFrom) and node.module: - module_name = node.module.split('.')[0] - + module_name = node.module.split(".")[0] + if module_name and module_name not in stdlib_names: - pip_name = COMMON_PIP_ALIASES.get(module_name, module_name) + pip_name = COMMON_PIP_ALIASES.get( + module_name, module_name + ) project_dependencies.add(pip_name) except Exception as e: - print(f"Warning: Failed to parse AST for block {script_name}: {e}") + print( + f"Warning: Failed to parse AST for block {script_name}: {e}" + ) + + script_name += dependency["package"]["version"].replace( + ".", "" + ) # Append version number to the script name, removing dots - script_name += dependency['package']['version'].replace('.', '') # Append version number to the script name, removing dots - # Increment the count for this script type or initialize it if script_name in dep_no: dep_no[script_name] += 1 else: dep_no[script_name] = 1 - - script_name += str(dep_no[script_name]) # Add the block count to the script name - dependencies[key] = script_name # Map the key to the script name in dependencies - - - zipfile.append(f'{BLOCK_DIRECTORY}/{script_name}.py', script) # Add the script to the zipfile - blocks[block_id] = {'name': script_name, 'type': block_type} # Store block information in the blocks dictionary + script_name += str( + dep_no[script_name] + ) # Add the block count to the script name + dependencies[key] = ( + script_name # Map the key to the script name in dependencies + ) + + zipfile.append( + f"{BLOCK_DIRECTORY}/{script_name}.py", script + ) # Add the script to the zipfile + blocks[block_id] = { + "name": script_name, + "type": block_type, + } # Store block information in the blocks dictionary # Check if the block is of type 'basic.constant' - elif block_type == 'basic.constant': - parameters[block_id] = parameters.get(block_id, []) # Initialize parameter list for the block if not already present + elif block_type == "basic.constant": + parameters[block_id] = parameters.get( + block_id, [] + ) # Initialize parameter list for the block if not already present # Append block's parameter data to the parameters dictionary - parameters[block_id].append({ - 'id': block['id'], - 'name': block['data']['name'], - 'value': block['data']['value']} + parameters[block_id].append( + { + "id": block["id"], + "name": block["data"]["name"], + "value": block["data"]["value"], + } ) # Recursively process nested dependencies if present - if 'dependencies' in dependency and dependency['dependencies']: - process_dependency(dependency['dependencies'], zipfile, synhronize_frequency, optional_files, dep_no, parameters, dependencies) + if "dependencies" in dependency and dependency["dependencies"]: + process_dependency( + dependency["dependencies"], + zipfile, + synhronize_frequency, + optional_files, + dep_no, + parameters, + dependencies, + ) # Process the top-level dependencies from the data - if 'dependencies' in data: - process_dependency(data['dependencies'], zipfile, synhronize_frequency, optional_files, dep_no, parameters, dependencies) + if "dependencies" in data: + process_dependency( + data["dependencies"], + zipfile, + synhronize_frequency, + optional_files, + dep_no, + parameters, + dependencies, + ) + + count = 1 # Initialize a counter for naming blocks - - count = 1 # Initialize a counter for naming blocks + # Iterate over the blocks in the main design graph + for block in data["design"]["graph"]["blocks"]: + if ( + "source" not in block and "target" not in block + ): # Skip blocks that have a 'source' or 'target' property - # Iterate over the blocks in the main design graph - for block in data['design']['graph']['blocks']: - - if 'source' not in block and 'target' not in block: # Skip blocks that have a 'source' or 'target' property - - block_id, block_type = block['id'], block['type'] + block_id, block_type = block["id"], block["type"] # Check if the block is of type 'basic.code' - if block_type == 'basic.code': - - code_name = "Code_" + str(count) # Generate a unique code name for the block + if block_type == "basic.code": + + code_name = "Code_" + str( + count + ) # Generate a unique code name for the block count += 1 # Increment the block counter - script = block['data']['code'] # Retrieve the script code from the block's data + script = block["data"][ + "code" + ] # Retrieve the script code from the block's data # Set the synchronization frequency, defaulting to 30 - synhronize_frequency[block_id] = get_number_or_default(block['data'].get('frequency', 30), 30) - zipfile.append(f'{BLOCK_DIRECTORY}/{code_name}.py', script) # Add the script to the zipfile with the generated code name - blocks[block_id] = {'name': code_name, 'type': block_type} # Store block information in the blocks dictionary + synhronize_frequency[block_id] = get_number_or_default( + block["data"].get("frequency", 30), 30 + ) + zipfile.append( + f"{BLOCK_DIRECTORY}/{code_name}.py", script + ) # Add the script to the zipfile with the generated code name + blocks[block_id] = { + "name": code_name, + "type": block_type, + } # Store block information in the blocks dictionary # Check if the block is of type 'basic.constant' - elif block_type == 'basic.constant': + elif block_type == "basic.constant": # Add block's parameter data to the parameters dictionary - parameters[block_id] = [{'name': block['data']['name'], 'value': block['data']['value']}] - - - valid_wires = [] # Initialize a list to store valid wire connections + parameters[block_id] = [ + {"name": block["data"]["name"], "value": block["data"]["value"]} + ] + valid_wires = [] # Initialize a list to store valid wire connections # Iterate over all wires in the wire component list for wire in wire_comp: # Retrieve source and target block IDs for the wire - source_id = wire['source']['block'] - target_id = wire['target']['block'] + source_id = wire["source"]["block"] + target_id = wire["target"]["block"] # Check if the source and target blocks are in the blocks or parameters dictionary source_in_blocks = source_id in blocks @@ -188,79 +258,110 @@ def process_dependency(dep, zipfile, synhronize_frequency, optional_files, dep_n target_in_parameters = target_id in parameters # Validate wires based on presence in blocks or parameters dictionaries - if (source_in_blocks and target_in_blocks) or \ - (source_in_parameters and target_in_parameters) or \ - (source_in_blocks and target_in_parameters) or \ - (source_in_parameters and target_in_blocks): + if ( + (source_in_blocks and target_in_blocks) + or (source_in_parameters and target_in_parameters) + or (source_in_blocks and target_in_parameters) + or (source_in_parameters and target_in_blocks) + ): # If valid, add the wire to the valid_wires list valid_wires.append(wire) else: # Mark source and target as 'absent' if not in blocks or parameters if source_id not in blocks and source_id not in parameters: - wire['source']['ob'] = 'absent' + wire["source"]["ob"] = "absent" if target_id not in blocks and target_id not in parameters: - wire['target']['ob'] = 'absent' + wire["target"]["ob"] = "absent" # Add the wire to the valid_wires list valid_wires.append(wire) - def process_wires(valid_wires): count = 0 # Initialize a counter for processing iterations - changes_detected = True # Boolean flag to track if changes were made in the current iteration + changes_detected = ( + True # Boolean flag to track if changes were made in the current iteration + ) # Loop until no changes are detected while changes_detected: - wire_check_source = {} # Dictionary to store wires' source info where 'ob' is absent - wire_check_target = {} - changes_detected = False # Reset changes_detected to False before processing wires + wire_check_source = ( + {} + ) # Dictionary to store wires' source info where 'ob' is absent + wire_check_target = {} + changes_detected = ( + False # Reset changes_detected to False before processing wires + ) new_wires = [] # List to store newly created wires # Iterate through valid_wires in reverse order for i in range(len(valid_wires) - 1, -1, -1): wire = valid_wires[i] # Access the current wire from valid_wires - remove_wire = False # Flag to mark whether the current wire should be removed + remove_wire = ( + False # Flag to mark whether the current wire should be removed + ) count += 1 # Increment the processing counter # Check if the source port is 'input-out' - if wire['source']['port'] == 'input-out': - port_name = wire['source']['block'] # Get the block name of the source port - + if wire["source"]["port"] == "input-out": + port_name = wire["source"][ + "block" + ] # Get the block name of the source port + # If the target has 'ob' and it's 'absent', track it for processing later - if 'ob' in wire['target'] and wire['target']['ob'] == 'absent': - if port_name not in wire_check_source: - wire_check_source[port_name] = [] # Initialize list if it's not present - wire_check_source[port_name].append(wire['target'].copy()) # Append a copy of the target - wire_check_source[port_name][-1]['port'] = port_name # Set the port name for the new entry + if "ob" in wire["target"] and wire["target"]["ob"] == "absent": + if port_name not in wire_check_source: + wire_check_source[port_name] = ( + [] + ) # Initialize list if it's not present + wire_check_source[port_name].append( + wire["target"].copy() + ) # Append a copy of the target + wire_check_source[port_name][-1][ + "port" + ] = port_name # Set the port name for the new entry else: # Otherwise, add the target to wire_check_source and mark for removal - if port_name not in wire_check_source: + if port_name not in wire_check_source: wire_check_source[port_name] = [] - wire_check_source[port_name].append(wire['target']) # Add target to the dictionary + wire_check_source[port_name].append( + wire["target"] + ) # Add target to the dictionary remove_wire = True # Mark the wire for removal later # Check if the target port is 'output-in' - if wire['target']['port'] == 'output-in': - port_name = wire['target']['block'] # Get the block name of the target port - + if wire["target"]["port"] == "output-in": + port_name = wire["target"][ + "block" + ] # Get the block name of the target port + # If the source has 'ob' and it's 'absent', track it for processing later - if 'ob' in wire['source'] and wire['source']['ob'] == 'absent': - if port_name not in wire_check_target: - wire_check_target[port_name] = [] # Initialize list if it's not present - wire_check_target[port_name].append(wire['source'].copy()) # Append a copy of the source - wire_check_target[port_name][-1]['port'] = port_name # Set the port name for the new entry + if "ob" in wire["source"] and wire["source"]["ob"] == "absent": + if port_name not in wire_check_target: + wire_check_target[port_name] = ( + [] + ) # Initialize list if it's not present + wire_check_target[port_name].append( + wire["source"].copy() + ) # Append a copy of the source + wire_check_target[port_name][-1][ + "port" + ] = port_name # Set the port name for the new entry else: # Otherwise, add the source to wire_check_target and mark for removal if port_name not in wire_check_target: wire_check_target[port_name] = [] - wire_check_target[port_name].append(wire['source']) # Add source to the dictionary + wire_check_target[port_name].append( + wire["source"] + ) # Add source to the dictionary remove_wire = True # Mark the wire for removal later # Remove the wire if it was marked for removal if remove_wire: del valid_wires[i] # Remove the wire from valid_wires - changes_detected = True # Set changes_detected to True as wires were modified + changes_detected = ( + True # Set changes_detected to True as wires were modified + ) i = 0 # Initialize counter for iterating through valid_wires new_wires = [] # Reset the list to store any newly created wires @@ -270,45 +371,65 @@ def process_wires(valid_wires): wire = valid_wires[i] # Access the current wire # Check if the source port is exactly 36 characters (for specific port length) - if len(wire['source'].get('port', '')) == 36: - port_name = wire['source']['port'] # Get the port name of the source + if len(wire["source"].get("port", "")) == 36: + port_name = wire["source"][ + "port" + ] # Get the port name of the source # If the port name exists in wire_check_target, process the sources if port_name in wire_check_target and wire_check_target[port_name]: - new_sources = wire_check_target[port_name] # Get the list of sources for the port - + new_sources = wire_check_target[ + port_name + ] # Get the list of sources for the port + # Replace the source of the current wire with the first new source - valid_wires[i]['source'] = new_sources[0] - if 'ob' in valid_wires[i]['source']: # If 'ob' exists, set the port name - valid_wires[i]['source']['port'] = port_name - + valid_wires[i]["source"] = new_sources[0] + if ( + "ob" in valid_wires[i]["source"] + ): # If 'ob' exists, set the port name + valid_wires[i]["source"]["port"] = port_name + # For remaining sources, create new wires for new_source in new_sources[1:]: new_wire = wire.copy() # Copy the current wire - new_wire['source'] = new_source # Set the new source for the copied wire - if 'ob' in new_wire['source']: # If 'ob' exists, set the port name - new_wire['source']['port'] = port_name + new_wire["source"] = ( + new_source # Set the new source for the copied wire + ) + if ( + "ob" in new_wire["source"] + ): # If 'ob' exists, set the port name + new_wire["source"]["port"] = port_name new_wires.append(new_wire) # Add the new wire to new_wires # Check if the target port is exactly 36 characters (for specific port length) - if len(wire['target'].get('port', '')) == 36: - port_name = wire['target']['port'] # Get the port name of the target - + if len(wire["target"].get("port", "")) == 36: + port_name = wire["target"][ + "port" + ] # Get the port name of the target + # If the port name exists in wire_check_source, process the targets if port_name in wire_check_source and wire_check_source[port_name]: - new_targets = wire_check_source[port_name] # Get the list of targets for the port - + new_targets = wire_check_source[ + port_name + ] # Get the list of targets for the port + # Replace the target of the current wire with the first new target - valid_wires[i]['target'] = new_targets[0] - if 'ob' in valid_wires[i]['target']: # If 'ob' exists, set the port name - valid_wires[i]['target']['port'] = port_name - + valid_wires[i]["target"] = new_targets[0] + if ( + "ob" in valid_wires[i]["target"] + ): # If 'ob' exists, set the port name + valid_wires[i]["target"]["port"] = port_name + # For remaining targets, create new wires for new_target in new_targets[1:]: new_wire = wire.copy() # Copy the current wire - new_wire['target'] = new_target # Set the new target for the copied wire - if 'ob' in new_wire['target']: # If 'ob' exists, set the port name - new_wire['target']['port'] = port_name + new_wire["target"] = ( + new_target # Set the new target for the copied wire + ) + if ( + "ob" in new_wire["target"] + ): # If 'ob' exists, set the port name + new_wire["target"]["port"] = port_name new_wires.append(new_wire) # Add the new wire to new_wires i += 1 # Move to the next wire in the valid_wires list @@ -323,8 +444,15 @@ def process_wires(valid_wires): # Iterate over valid_wires to remove duplicates for wire in valid_wires: # Convert the wire dictionary to a frozenset for hashability (so it can be added to a set) - wire_tuple = frozenset((key, frozenset(value.items())) if isinstance(value, dict) else (key, value) for key, value in wire.items()) - + wire_tuple = frozenset( + ( + (key, frozenset(value.items())) + if isinstance(value, dict) + else (key, value) + ) + for key, value in wire.items() + ) + # Only add unique wires to unique_wires if wire_tuple not in seen_wires: seen_wires.add(wire_tuple) # Add the wire to the set of seen wires @@ -334,71 +462,79 @@ def process_wires(valid_wires): return valid_wires # Return the processed valid_wires - # Call the process_wires function to process the wires processed_wires = process_wires(valid_wires) # Package processed data into a JSON file for saving or further use data = { - 'blocks': blocks, # Block-related data - 'parameters': parameters, # Parameter-related data - 'synchronize_frequency': synhronize_frequency, # Synchronization frequency information - 'wires': processed_wires # The processed wire data + "blocks": blocks, # Block-related data + "parameters": parameters, # Parameter-related data + "synchronize_frequency": synhronize_frequency, # Synchronization frequency information + "wires": processed_wires, # The processed wire data } # Add data to a zipfile (the zipfile object must be defined elsewhere in the code) - zipfile.append('data.json', json.dumps(data)) # Append the JSON data to the zipfile - + zipfile.append("data.json", json.dumps(data)) # Append the JSON data to the zipfile return zipfile, optional_files, project_dependencies -def synthesize_executioner(zipfile: InMemoryZip, optional_files: Dict[str, bool]) -> InMemoryZip: - '''Synthesize python code necessary to run the blocks. - All these files are present in django static directory. + +def synthesize_executioner( + zipfile: InMemoryZip, optional_files: Dict[str, bool] +) -> InMemoryZip: + """Synthesize python code necessary to run the blocks. + All these files are present in django static directory. They are read and put into the zipfile. - ''' + """ # Blacklist all optional files by default paths_to_exclude = set(OPTIONAL_FILES.values()) # If a particular block is present which needs optional files, whitelist the required optional files. - paths_to_include = set([path for key, path in OPTIONAL_FILES.items() if optional_files.get(key, False)]) - - for path in get_files(staticfiles_storage, location='synthesis'): + paths_to_include = set( + [path for key, path in OPTIONAL_FILES.items() if optional_files.get(key, False)] + ) + + for path in get_files(staticfiles_storage, location="synthesis"): with staticfiles_storage.open(path) as file: content = file.read() - relative_path = Path(path).relative_to('synthesis') + relative_path = Path(path).relative_to("synthesis") # Check if the path is excluded, if it is, check if its required for the current set of blocks. - if not any([relative_path.match(p) for p in paths_to_exclude]) or any([relative_path.match(p) for p in paths_to_include]): - zipfile.append(str(relative_path), content) + if not any([relative_path.match(p) for p in paths_to_exclude]) or any( + [relative_path.match(p) for p in paths_to_include] + ): + zipfile.append(str(relative_path), content) return zipfile + def syntesize_extras(zipfile: InMemoryZip) -> InMemoryZip: - '''Create and extra files which might be required for execution. - ''' - zipfile.append('logs/console.log', '') + """Create and extra files which might be required for execution.""" + zipfile.append("logs/console.log", "") return zipfile + def synthesize(data: dict) -> Tuple[str, BytesIO]: - '''Synthesize a python application corresponding to the VC project file. + """Synthesize a python application corresponding to the VC project file. All synthesized files are put inside a zip file so that it can be downloaded. - ''' + """ zipfile = InMemoryZip() # Optional files required based on blocks present. zipfile, optional_files, project_dependencies = syntheize_modules(data, zipfile) zipfile = synthesize_executioner(zipfile, optional_files) zipfile = syntesize_extras(zipfile) - + if project_dependencies: requirements_content = "\n".join(project_dependencies) + "\n" - zipfile.append('requirements.txt', requirements_content) + zipfile.append("requirements.txt", requirements_content) - # Project name (zipfile name) - project_name = f"{data['package']['name']}" if data['package']['name'] != '' else 'Project' + # Project name (zipfile name) + project_name = ( + f"{data['package']['name']}" if data["package"]["name"] != "" else "Project" + ) # Add the .vc3 file to the built application, this will let us easily load the project in Visual Circuit zipfile.append(project_name + PROJECT_FILE_EXTENSION, json.dumps(data)) # .zip is required for the name of the full package - project_name += '.zip' + project_name += ".zip" - return project_name, zipfile.get_zip() \ No newline at end of file + return project_name, zipfile.get_zip()