source /home/vivek/Git/ska-referee-test/ref-test/env/bin/activateMerge branch 'development'

.env.example

@@ -12,6 +12,7 @@ DATABASE_PORT= # Required if MySQL. Defaults to 3306
 
 ## MySQL Database Configuration (Required if configured to MySQL Database.)
 # Note that if using the Docker service, these configuration values will also be used when creating the database in the mysql container.
+MYSQL_RANDOM_ROOT_PASSWORD=True
 MYSQL_DATABASE= # Required if MySQL.
 MYSQL_USER= # Required if MySQL
 MYSQL_PASSWORD= # Required if MySQL. Create secure password string. Note '@' character cannot be used.

.gitignore

@@ -152,4 +152,14 @@ database/data/
 .encryption.key
 
 # Ignore Data Dir
-**/data/*
+**/data/*
+
+# Ignore Logs Dir
+logs/*
+
+# Ignore Certbot Dir
+certbot/*
+
+# Ignore src dir (exception for robots.txt)
+src/html/*
+src/html/robots.txt

README.md

@@ -57,6 +57,16 @@ Once in the destination folder, clone all the relevant files you will need for t
 
 (Remember to include the trailing dot at the end, as that indicates to Git to download the files in the current directory.)
 
+#### Choose What Database Engine You Will Use
+
+This app is designed to use an SQLite database by default.
+You can set it up to use a MySQL database by configuring the environment variables accordingly.
+If your database is being hosted remotely, make sure the MySQL database has the proper authentication for the user from a remote server.
+Alternatively, you can also use the second `docker-compose-mysql.yml` file which provides a MySQL database as part of the cluster.
+To use the second `docker-compose-mysql.yml` file, use the following command at the last step of the installation:
+
+```sudo docker compose -f docker-compose-mysql.yml up```
+
 #### Populate Environment Variables
 
 Configuration values for the app are stored in the environment variables file.

certbot/.gitignore

@@ -1,2 +0,0 @@
-*
-!.gitignore

docker-compose-mysql.yml

@@ -0,0 +1,90 @@
+version: '3.9'
+
+volumes:
+  app:
+  mysql:
+
+services:
+  nginx:
+    container_name: reftest_server
+    image: nginx:alpine
+    volumes:
+      - ./certbot:/etc/letsencrypt:ro
+      - ./nginx:/etc/nginx
+      - ./src/html/certbot:/usr/share/nginx/html/certbot:ro
+      - ./src/html/robots.txt:/usr/share/nginx/html/robots.txt:ro
+      - ./ref-test/app/root:/usr/share/nginx/html/root:ro
+      - ./ref-test/app/admin/static:/usr/share/nginx/html/admin/static:ro
+      - ./ref-test/app/editor/static:/usr/share/nginx/html/editor/static:ro
+      - ./ref-test/app/quiz/static:/usr/share/nginx/html/quiz/static:ro
+      - ./ref-test/app/view/static:/usr/share/nginx/html/view/static:ro
+    ports:
+      - 80:80
+      - 443:443
+    restart: unless-stopped
+    networks:
+      - frontend
+    depends_on:
+      - app
+    command: "/bin/sh -c 'while :; do sleep 6h & wait $${!}; nginx -s reload; done & nginx -g \"daemon off;\"'"
+  
+  app:
+    container_name: reftest_app
+    image: reftest
+    build: ./ref-test
+    env_file:
+      - ./.env
+    ports:
+      - 5000
+    volumes:
+      - app:/ref-test/data
+      - ./logs:/ref-test/data/logs
+    restart: unless-stopped
+    networks:
+      - frontend
+      - backend
+    depends_on:
+      postfix:
+      mysql:
+        condition: service_healthy
+
+  postfix:
+    container_name: reftest_postfix
+    image: catatnight/postfix:latest
+    restart: unless-stopped
+    env_file:
+      - ./.env
+    ports:
+      - 25
+    networks:
+      - backend
+  
+  certbot:
+    container_name: reftest_certbot
+    image: certbot/certbot
+    volumes:
+      - ./certbot:/etc/letsencrypt
+      - ./src/html/certbot:/var/www/html
+    entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
+  
+  mysql:
+    container_name: reftest_db
+    image: mysql:8.0
+    env_file:
+      - ./.env
+    volumes:
+      - mysql:/var/lib/mysql
+    ports:
+      - 3306
+    networks:
+      - backend
+    healthcheck:
+      test: ["CMD", "mysqladmin" ,"ping", "-h", "localhost"]
+      timeout: 10s
+      retries: 10
+
+networks:
+  frontend:
+    external: false
+  backend:
+    external: false

docker-compose.yml

@@ -1,7 +1,7 @@
 version: '3.9'
 
 volumes:
-  data:
+  app:
 
 services:
   nginx:
@@ -10,6 +10,7 @@ services:
     volumes:
       - ./certbot:/etc/letsencrypt:ro
       - ./nginx:/etc/nginx
+      - ./src/html/certbot:/usr/share/nginx/html/certbot:ro
       - ./src/html/robots.txt:/usr/share/nginx/html/robots.txt:ro
       - ./ref-test/app/root:/usr/share/nginx/html/root:ro
       - ./ref-test/app/admin/static:/usr/share/nginx/html/admin/static:ro
@@ -35,7 +36,8 @@ services:
     ports:
       - 5000
     volumes:
-      - data:/ref-test/data
+      - app:/ref-test/data
+      - ./logs:/ref-test/data/logs
     restart: unless-stopped
     networks:
       - frontend
@@ -59,7 +61,7 @@ services:
     image: certbot/certbot
     volumes:
       - ./certbot:/etc/letsencrypt
-      - ./src/html:/var/www/html
+      - ./src/html/certbot:/var/www/html
     entrypoint: "/bin/sh -c 'trap exit TERM; while :; do certbot renew; sleep 12h & wait $${!}; done;'"
   
 networks:

nginx/certbot-challenge.conf

@@ -1,6 +1,6 @@
 # Certbot Renewal
 location ^~ /.well-known/acme-challenge/ {
-    root /usr/share/nginx/html;
+    root /usr/share/nginx/html/certbot;
     allow all;
     default_type "text/plain";
 }

ref-test/.dockerignore

@@ -1,2 +1,3 @@
 env/
-__pycache__/
+__pycache__/
+data/

ref-test/app/__init__.py

@@ -1,5 +1,5 @@
 from .config import Production as Config
-from .models import User
+from .models import *
 from .extensions import bootstrap, csrf, db, login_manager, mail
 from .tools.logs import write
 
@@ -7,7 +7,6 @@ from flask import flash, Flask, render_template, request
 from flask.helpers import abort, url_for
 from flask.json import jsonify
 from flask_wtf.csrf import CSRFError
-from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.middleware.proxy_fix import ProxyFix
 
 from datetime import datetime
@@ -27,7 +26,7 @@ def create_app():
     @login_manager.user_loader
     def _load_user(id):
         try: return User.query.filter_by(id=id).first()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when loading user fo login manager: {exception}')
             return abort(500)
 
@@ -59,5 +58,11 @@ def create_app():
     app.register_blueprint(quiz)
     app.register_blueprint(editor, url_prefix='/admin/editor')
     app.register_blueprint(view, url_prefix='/admin/view')
-    
+
+    """Create Database Tables before First Request"""
+    @app.before_first_request
+    def _create_database_tables():
+        with app.app_context():
+            db.create_all()
+
     return app

ref-test/app/admin/templates/admin/components/navbar.html

@@ -1,6 +1,6 @@
 <nav class="navbar fixed-top navbar-expand-md navbar-dark bg-dark">
     <div class="container">
-        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest (Beta) | Admin</a>
+        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest | Admin</a>
         <button
             class="navbar-toggler"
             type="button"

ref-test/app/admin/templates/admin/test.html

@@ -19,7 +19,7 @@
                         <div class="d-flex w-100 justify-content-between">
                             <h5 class="mb-1">Dataset</h5>
                         </div>
-                        {{ test.dataset.date.strftime('%Y%m%d%H%M%S') }}
+                        <a href="{{ url_for('view._view_console', id=test.dataset.id) }}">{{ test.dataset.get_name() }}</a>
                     </li>
                     <li class="list-group-item list-group-item-action">
                         <div class="d-flex w-100 justify-content-between">

ref-test/app/admin/views.py

@@ -9,7 +9,6 @@ from ..tools.test import  answer_options, get_correct_answers
 from flask import abort, Blueprint, jsonify, render_template, request, send_file, session
 from flask.helpers import abort, flash, redirect, url_for
 from flask_login import current_user, login_required
-from sqlalchemy.exc import SQLAlchemyError
 
 from datetime import date, datetime, timedelta
 from json import loads
@@ -31,7 +30,7 @@ def _home():
     try:
         tests = Test.query.all()
         results = Entry.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     current_tests = [ test for test in tests if test.end_date >= datetime.now() and test.start_date.date() <= date.today() ]
@@ -48,7 +47,7 @@ def _settings():
     try:
         users = User.query.all()
         datasets = Dataset.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     return render_template('/admin/settings/index.html', users=users, datasets=datasets)
@@ -61,7 +60,7 @@ def _login():
     if request.method == 'POST':
         if form.validate_on_submit():
             try: users = User.query.all()
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
                 return abort(500)
             user = None
@@ -113,7 +112,7 @@ def _reset():
         if form.validate_on_submit():
             user = None
             try: users = User.query.all()
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
                 return abort(500)
             for _user in users:
@@ -128,7 +127,7 @@ def _reset():
     token = request.args.get('token')
     if token:
         try: user = User.query.filter_by(reset_token=token).first()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
             return abort(500)
         if not user: return redirect(url_for('admin._reset'))
@@ -148,7 +147,7 @@ def _update_password():
     if form.validate_on_submit():
         user = session.pop('user')
         try: user = User.query.filter_by(id=user).first()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
             return abort(500)
         user.update(password=request.form.get('password'))
@@ -162,7 +161,7 @@ def _update_password():
 def _users():
     form = CreateUser()
     try: users = User.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if request.method == 'POST':
@@ -182,7 +181,7 @@ def _users():
 @login_required
 def _delete_user(id:str):
     try: user = User.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     form = DeleteUser()
@@ -209,14 +208,14 @@ def _delete_user(id:str):
 @login_required
 def _update_user(id:str):
     try: user = User.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     form = UpdateUser()
     if request.method == 'POST': 
         if not user: return jsonify({'error': 'User does not exist.'}), 400
         if form.validate_on_submit():
-            if not user.verify_password(request.form.get('confirm_password')): return jsonify({'error': 'Invalid password for your account.'}), 401
+            if not current_user.verify_password(request.form.get('confirm_password')): return jsonify({'error': 'Invalid password for your account.'}), 401
             success, message = user.update(
                 password = request.form.get('password'),
                 email = request.form.get('email'),
@@ -254,7 +253,7 @@ def _questions():
         return send_errors_to_client(form=form)
 
     try: data = Dataset.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     return render_template('/admin/settings/questions.html', form=form, data=data)
@@ -266,7 +265,7 @@ def _edit_questions():
     action = request.get_json()['action']
     if not action == 'delete': return jsonify({'error': 'Invalid action.'}), 400
     try: dataset = Dataset.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if action == 'delete': success, message = dataset.delete()
@@ -277,7 +276,7 @@ def _edit_questions():
 @login_required
 def _download(id:str):
     try: dataset = Dataset.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not dataset: return abort(404)
@@ -291,7 +290,7 @@ def _download(id:str):
 def _tests(filter:str=None):
     tests = None
     try: _tests = Test.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     form = None
@@ -340,7 +339,7 @@ def _create_test():
         new_test.time_limit = None if request.form.get('time_limit') == 'none' else int(request.form.get('time_limit'))
         dataset = request.form.get('dataset')
         try: new_test.dataset = Dataset.query.filter_by(id=dataset).first()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
             return abort(500)
         success, message = new_test.create()
@@ -357,7 +356,7 @@ def _edit_test():
     action = request.get_json()['action']
     if action not in ['start', 'delete', 'end']: return jsonify({'error': 'Invalid action.'}), 400
     try: test = Test.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not test: return jsonify({'error': 'Could not find the corresponding test to delete.'}), 404
@@ -374,7 +373,7 @@ def _edit_test():
 def _view_test(id:str=None):    
     form = AddTimeAdjustment()
     try: test = Test.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if request.method == 'POST':
@@ -394,7 +393,7 @@ def _view_test(id:str=None):
 @login_required
 def _delete_adjustment(id:str=None):
     try: test = Test.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not test: return jsonify({'error': 'Invalid test ID.'}), 404
@@ -407,7 +406,7 @@ def _delete_adjustment(id:str=None):
 @login_required
 def _view_entries():
     try: entries = Entry.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     return render_template('/admin/results.html', entries = entries)
@@ -416,7 +415,7 @@ def _view_entries():
 @login_required
 def _view_entry(id:str=None):
     try: entry = Entry.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if request.method == 'POST':
@@ -450,7 +449,7 @@ def _generate_certificate():
     from ..extensions import db
     id = request.get_json()['id']
     try: entry = Entry.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not entry: return jsonify({'error': 'Invalid entry ID.'}), 404

ref-test/app/api/views.py

@@ -6,7 +6,6 @@ from ..tools.test import evaluate_answers, generate_questions
 from flask import Blueprint, jsonify, request
 from flask.helpers import abort, flash, url_for
 from flask_login import login_required
-from sqlalchemy.exc import SQLAlchemyError
 
 from datetime import datetime, timedelta
 from json import loads
@@ -20,7 +19,7 @@ api = Blueprint(
 def _fetch_questions():
     id = request.get_json()['id']
     try: entry = Entry.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not entry: return jsonify({'error': 'Invalid entry ID.'}), 400
@@ -57,7 +56,7 @@ def _submit_quiz():
     id = request.get_json()['id']
     answers = request.get_json()['answers']
     try: entry = Entry.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not entry: return jsonify({'error': 'Unrecognised Entry.'}), 400
@@ -80,10 +79,8 @@ def _submit_quiz():
 def _editor(id:str=None):
     request_data = request.get_json()
     id = request_data['id']
-    try:
-        dataset = Dataset.query.filter_by(id=id).first()
-        user = User.query.filter_by(id=creator).first()
-    except SQLAlchemyError as exception:
+    try: dataset = Dataset.query.filter_by(id=id).first()
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not dataset: return jsonify({'error': 'Invalid request. Dataset not found.'}), 404
@@ -94,6 +91,10 @@ def _editor(id:str=None):
         return jsonify({'success': 'Successfully downloaded dataset', 'data': data}), 200
     default = request_data['default']
     creator = request_data['creator']
+    try: user = User.query.filter_by(id=creator).first()
+    except Exception as exception:
+        write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
+        return abort(500)
     name = request_data['name']
     data = request_data['data']
     if not validate_json(data): return jsonify({'error': 'The data you submitted was invalid.'}), 400

ref-test/app/config.py

@@ -37,7 +37,7 @@ class Config(object):
         MYSQL_USER = os.getenv('MYSQL_USER')
         MYSQL_PASSWORD = os.getenv('MYSQL_PASSWORD')
         SQLALCHEMY_DATABASE_URI = f'mysql+pymysql://{MYSQL_USER}:{MYSQL_PASSWORD}@{DATABASE_HOST}:{DATABASE_PORT}/{MYSQL_DATABASE}'
-    else: SQLALCHEMY_DATABASE_URI = f'sqlite:///{Path(os.path.abspath(f"{DATA}/database.db"))}'
+    else: SQLALCHEMY_DATABASE_URI = f'sqlite:///{Path(os.path.abspath(f"{DATA}/db.sqlite"))}'
 
 class Production(Config):
     pass

ref-test/app/editor/templates/editor/components/navbar.html

@@ -1,6 +1,6 @@
 <nav class="navbar fixed-top navbar-expand-md navbar-dark bg-dark">
     <div class="container">
-        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest (Beta) | Admin</a>
+        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest | Admin</a>
         <button
             class="navbar-toggler"
             type="button"

ref-test/app/editor/views.py

@@ -7,7 +7,6 @@ from ..tools.logs import write
 from flask import Blueprint, jsonify, render_template
 from flask.helpers import abort, flash, redirect, request, url_for
 from flask_login import login_required
-from sqlalchemy.exc import SQLAlchemyError
 
 editor = Blueprint(
     name='editor',
@@ -37,7 +36,7 @@ def _editor_console(id:str=None):
         dataset = Dataset.query.filter_by(id=id).first()
         datasets = Dataset.query.count()
         users = User.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not dataset:

ref-test/app/models/dataset.py

@@ -5,7 +5,6 @@ from ..tools.logs import write
 from flask import current_app as app
 from flask.helpers import flash
 from flask_login import current_user
-from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.utils import secure_filename
 
 from datetime import datetime
@@ -15,8 +14,7 @@ from pathlib import Path
 from uuid import uuid4
 
 class Dataset(db.Model):
-
-    id = db.Column(db.String(36), primary_key=True)
+    id = db.Column(db.String(36), index=True, primary_key=True)
     name = db.Column(db.String(128), nullable=False)
     tests = db.relationship('Test', backref='dataset')
     creator_id = db.Column(db.String(36), db.ForeignKey('user.id'))
@@ -45,12 +43,12 @@ class Dataset(db.Model):
     def make_default(self):
         try:
             for dataset in Dataset.query.all(): dataset.default = False
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when setting default dataset {self.id}: {exception}')
             return False, f'Database error {exception}.'
         self.default = True
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when setting default dataset {self.id}: {exception}')
             return False, f'Database error {exception}.'
@@ -68,7 +66,7 @@ class Dataset(db.Model):
                 message = 'Cannot delete the only dataset.'
                 flash(message, 'error')
                 return False, message
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when setting default dataset {self.id}: {exception}')
             return False, f'Database error {exception}.'
         write('system.log', f'Dataset {self.id} deleted by {current_user.get_username()}.')
@@ -78,7 +76,7 @@ class Dataset(db.Model):
         try:
             db.session.delete(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when trying to delete dataset {self.id}: {exception}')
             return False, f'Database error: {exception}'
@@ -98,7 +96,7 @@ class Dataset(db.Model):
         try:
             db.session.add(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when trying to crreate dataset {self.id}: {exception}')
             return False, f'Database error: {exception}'
@@ -128,7 +126,7 @@ class Dataset(db.Model):
         try:
             db.session.add(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when trying to update dataset {self.id}: {exception}')
             return False, f'Database error: {exception}'

ref-test/app/models/entry.py

@@ -1,32 +1,29 @@
 from ..extensions import db, mail
-from ..tools.forms import JsonEncodedDict
 from ..tools.encryption import decrypt, encrypt
 from ..tools.logs import write
 from .test import Test
 
 from flask_login import current_user
 from flask_mail import Message
-from smtplib import SMTPException
-from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy_json import MutableJson
 
 from datetime import datetime, timedelta
 from uuid import uuid4
 
 class Entry(db.Model):
-
-    id = db.Column(db.String(36), primary_key=True)
+    id = db.Column(db.String(36), index=True, primary_key=True)
     first_name = db.Column(db.String(128), nullable=False)
     surname = db.Column(db.String(128), nullable=False)
     email = db.Column(db.String(128), nullable=False)
     club = db.Column(db.String(128), nullable=True)
     test_id = db.Column(db.String(36), db.ForeignKey('test.id'))
     user_code = db.Column(db.String(6), nullable=True)
-    start_time = db.Column(db.DateTime, nullable=True)
-    end_time = db.Column(db.DateTime, nullable=True)
+    start_time = db.Column(db.DateTime, index=True, nullable=True)
+    end_time = db.Column(db.DateTime, index=True, nullable=True)
     status = db.Column(db.String(16), nullable=True)
     valid = db.Column(db.Boolean, default=True, nullable=True)
-    answers = db.Column(JsonEncodedDict, nullable=True)
-    result = db.Column(JsonEncodedDict, nullable=True)
+    answers = db.Column(MutableJson, nullable=True)
+    result = db.Column(MutableJson, nullable=True)
     
     def __repr__(self):
         return f'<New entry by {self.first_name} {self.surname}> was added with <id {self.id}>.'
@@ -74,7 +71,7 @@ class Entry(db.Model):
         try:
             db.session.add(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when preparing new entry for {self.get_surname()}, {self.get_first_name()}: {exception}')
             return False, f'Database error: {exception}'
@@ -85,7 +82,7 @@ class Entry(db.Model):
         self.start_time = datetime.now()
         self.status = 'started'
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when starting test for {self.get_surname()}, {self.get_first_name()}: {exception}')
             return False, f'Database error: {exception}'
@@ -104,7 +101,7 @@ class Entry(db.Model):
             self.status = 'late'
             self.valid = False
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when submitting entry for {self.get_surname()}, {self.get_first_name()}: {exception}')
             return False, f'Database error: {exception}'
@@ -117,7 +114,7 @@ class Entry(db.Model):
         self.valid = True
         self.status = 'completed'
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when validating entry {self.id}: {exception}')
             return False, f'Database error: {exception}'
@@ -130,7 +127,7 @@ class Entry(db.Model):
         try:
             db.session.delete(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when deleting entry {id}: {exception}')
             return False, f'Database error: {exception}'
@@ -199,5 +196,4 @@ class Entry(db.Model):
             """
         )
         try: mail.send(email)
-        except SMTPException as exception:
-            write('system.log', f'SMTP Error when trying to notify results to {self.get_surname()}, {self.get_first_name()} with error: {exception}')
+        except Exception as exception: write('system.log', f'SMTP Error when trying to notify results to {self.get_surname()}, {self.get_first_name()} with error: {exception}')

ref-test/app/models/test.py

@@ -1,24 +1,22 @@
 from ..extensions import db
-from ..tools.forms import JsonEncodedDict
 from ..tools.logs import write
 
 from flask_login import current_user
-from sqlalchemy.exc import SQLAlchemyError
+from sqlalchemy_json import MutableJson
 
 from datetime import date, datetime
 import secrets
 from uuid import uuid4
 
 class Test(db.Model):
-    
-    id = db.Column(db.String(36), primary_key=True)
-    code = db.Column(db.String(36), nullable=False)
-    start_date = db.Column(db.DateTime, nullable=True)
+    id = db.Column(db.String(36), index=True, primary_key=True)
+    code = db.Column(db.String(36), index=True, nullable=False)
+    start_date = db.Column(db.DateTime, index=True, nullable=True)
     end_date = db.Column(db.DateTime, nullable=True)
     time_limit = db.Column(db.Integer, nullable=True)
     creator_id = db.Column(db.String(36), db.ForeignKey('user.id'))
     dataset_id = db.Column(db.String(36), db.ForeignKey('dataset.id'))
-    adjustments = db.Column(JsonEncodedDict, nullable=True)
+    adjustments = db.Column(MutableJson, nullable=True)
     entries = db.relationship('Entry', backref='test')
 
     def __repr__(self):
@@ -56,7 +54,7 @@ class Test(db.Model):
         try:
             db.session.add(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when creating test {self.get_code()}: {exception}')
             return False, f'Database error: {exception}'
@@ -67,7 +65,7 @@ class Test(db.Model):
         if self.entries: return False, f'Cannot delete a test with submitted entries.'
         db.session.delete(self)
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when deleting test {self.get_code()}: {exception}')
             return False, f'Database error: {exception}'
@@ -79,7 +77,7 @@ class Test(db.Model):
         if self.start_date.date() > now.date():
             self.start_date = now
             try: db.session.commit()
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 db.session.rollback()
                 write('system.log', f'Database error when launching test {self.get_code()}: {exception}')
                 return False, f'Database error: {exception}'
@@ -92,7 +90,7 @@ class Test(db.Model):
         if self.end_date >= now:
             self.end_date = now
             try: db.session.commit()
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 db.session.rollback()
                 write('system.log', f'Database error when closing test {self.get_code()}: {exception}')
                 return False, f'Database error: {exception}'
@@ -106,7 +104,7 @@ class Test(db.Model):
         adjustments[code] = time
         self.adjustments = adjustments
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when adding adjustment to test {self.get_code()}: {exception}')
             return False, f'Database error: {exception}'
@@ -118,7 +116,7 @@ class Test(db.Model):
         self.adjustments.pop(code)
         if not self.adjustments: self.adjustments = None
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when deleting adjustment from test {self.get_code()}: {exception}')
             return False, f'Database error: {exception}'
@@ -131,7 +129,7 @@ class Test(db.Model):
         if end_date: self.end_date = end_date
         if time_limit is not None: self.time_limit = time_limit
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when updating test {self.get_code()}: {exception}')
             return False, f'Database error: {exception}'

ref-test/app/models/user.py

@@ -6,18 +6,16 @@ from flask import jsonify, session
 from flask.helpers import flash, url_for
 from flask_login import current_user, login_user, logout_user, UserMixin
 from flask_mail import Message
-from smtplib import SMTPException
-from sqlalchemy.exc import SQLAlchemyError
 from werkzeug.security import check_password_hash, generate_password_hash
 
 import secrets
 from uuid import uuid4
 class User(UserMixin, db.Model):
-    id = db.Column(db.String(36), primary_key=True)
+    id = db.Column(db.String(36), index=True, primary_key=True)
     username = db.Column(db.String(128), nullable=False)
     password = db.Column(db.String(128), nullable=False)
     email = db.Column(db.String(128), nullable=False)
-    reset_token = db.Column(db.String(20), nullable=True)
+    reset_token = db.Column(db.String(20), index=True, nullable=True)
     verification_token = db.Column(db.String(20), nullable=True)
     tests = db.relationship('Test', backref='creator')
     datasets = db.relationship('Dataset', backref='creator')
@@ -58,7 +56,7 @@ class User(UserMixin, db.Model):
     def register(self, notify:bool=False, password:str=None):
         self.generate_id()
         try: users = User.query.all()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when setting default dataset {self.id}: {exception}')
             return False, f'Database error {exception}.'
         for user in users:
@@ -68,7 +66,7 @@ class User(UserMixin, db.Model):
         try:
             db.session.add(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when registering user {self.get_username()}: {exception}')
             return False, f'Database error: {exception}'
@@ -101,8 +99,7 @@ class User(UserMixin, db.Model):
                 """
             )
             try: mail.send(email)
-            except SMTPException as exception:
-                write('system.log', f'SMTP Error while trying to notify new user account creation to {self.get_username()} with error: {exception}')
+            except Exception as exception: write('system.log', f'SMTP Error while trying to notify new user account creation to {self.get_username()} with error: {exception}')
         return True, f'User {self.get_username()} was created successfully.'
 
     def login(self, remember:bool=False):
@@ -154,12 +151,12 @@ class User(UserMixin, db.Model):
             """
         )
         try: mail.send(email)
-        except SMTPException as exception:
+        except Exception as exception:
             write('system.log', f'SMTP Error while trying to reset password for {self.get_username()} with error: {exception}')
             db.session.rollback()
             return jsonify({'error': f'SMTP Error: {exception}'}), 500
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when resetting password for user {self.get_username()}: {exception}')
             return False, f'Database error: {exception}'
@@ -168,7 +165,7 @@ class User(UserMixin, db.Model):
     def clear_reset_tokens(self):
         self.reset_token = self.verification_token = None
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when resetting clearing reset tokens for user {self.get_username()}: {exception}')
             return False, f'Database error: {exception}'
@@ -179,7 +176,7 @@ class User(UserMixin, db.Model):
         try:
             db.session.delete(self)
             db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when deleting user {self.get_username()}: {exception}')
             return False, f'Database error: {exception}'
@@ -208,8 +205,7 @@ class User(UserMixin, db.Model):
                 """
             )
             try: mail.send(email)
-            except SMTPException as exception:
-                write('system.log', f'SMTP Error when trying to delete account {username} with error: {exception}')
+            except Exception as exception: write('system.log', f'SMTP Error when trying to delete account {username} with error: {exception}')
         return True, message
 
     def update(self, password:str=None, email:str=None, notify:bool=False):
@@ -220,12 +216,12 @@ class User(UserMixin, db.Model):
             try:
                 for entry in User.query.all():
                     if entry.get_email() == email and not entry == self: return False, f'The email address {email} is already in use.'
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 write('system.log', f'Database error when setting default dataset {self.id}: {exception}')
                 return False, f'Database error {exception}.'
             self.set_email(email)
         try: db.session.commit()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             db.session.rollback()
             write('system.log', f'Database error when updating user {self.get_username()}: {exception}')
             return False, f'Database error: {exception}'
@@ -259,6 +255,5 @@ class User(UserMixin, db.Model):
                 """
             )
             try: mail.send(message)
-            except SMTPException as exception:
-                write('system.log', f'SMTP Error when trying to update account {self.get_username()} with error: {exception}')
+            except Exception as exception: write('system.log', f'SMTP Error when trying to update account {self.get_username()} with error: {exception}')
         return True, f'Account {self.get_username()} has been updated.'

ref-test/app/quiz/templates/quiz/components/base.html

@@ -17,7 +17,7 @@
         />
         {% block style %}
         {% endblock %}
-        <title>{% block title %} SKA Referee Test Beta {% endblock %}</title>
+        <title>{% block title %} SKA Referee Test {% endblock %}</title>
         {% include "quiz/components/og-meta.html" %}
     </head>
     <body class="bg-light">

ref-test/app/quiz/templates/quiz/components/navbar.html

@@ -1,6 +1,6 @@
 <nav class="navbar fixed-top navbar-expand-md navbar-dark bg-dark" id="primary-nav">
     <div class="container">
-        <p class="navbar-brand mb-0 h1">SKA Refereeing Test (Beta)</p>
+        <p class="navbar-brand mb-0 h1">SKA Refereeing Test</p>
         <div class="quiz-console w-100" style="display: none;" id="q-topbar">
             <div class="d-flex justify-content align-middle">
                 <div class="container d-flex justify-content-center">

ref-test/app/quiz/views.py

@@ -6,7 +6,6 @@ from ..tools.test import redirect_if_started
 
 from flask import Blueprint, jsonify, render_template, request, session
 from flask.helpers import abort, flash, redirect, url_for
-from sqlalchemy.exc import SQLAlchemyError
 
 from datetime import datetime
 
@@ -40,7 +39,7 @@ def _start():
             entry.set_email(request.form.get('email'))
             code = request.form.get('test_code').replace('—', '').lower()
             try: test = Test.query.filter_by(code=code).first()
-            except SQLAlchemyError as exception:
+            except Exception as exception:
                 write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
                 return abort(500)
             entry.test = test
@@ -69,7 +68,7 @@ def _quiz():
             flash('Your session was not recognised. Please sign in to the quiz again.', 'error')
             session.pop('id', None)
             return redirect(url_for('quiz._start'))
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     return render_template('/quiz/client.html')
@@ -78,7 +77,7 @@ def _quiz():
 def _result():
     id = session.get('id')
     try: entry = Entry.query.filter_by(id=id).first()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
         return abort(500)
     if not entry: return abort(404)

ref-test/app/templates/404.html

@@ -18,7 +18,7 @@
         <link rel="shortcut icon" href="{{ url_for('views.static', filename='favicon.ico') }}">
         {% block style %}
         {% endblock %}
-        <title>{% block title %} SKA Referee Test Beta {% endblock %}</title>
+        <title>{% block title %} SKA Referee Test {% endblock %}</title>
     </head>
     <body class="bg-light">
 

ref-test/app/templates/components/base.html

@@ -17,7 +17,7 @@
         />
         {% block style %}
         {% endblock %}
-        <title>{% block title %} SKA Referee Test Beta {% endblock %}</title>
+        <title>{% block title %} SKA Referee Test {% endblock %}</title>
         {% include "components/og-meta.html" %}
     </head>
     <body class="bg-light">

ref-test/app/templates/components/navbar.html

@@ -1,6 +1,6 @@
 <nav class="navbar fixed-top navbar-expand-md navbar-dark bg-dark" id="primary-nav">
     <div class="container">
-        <p class="navbar-brand mb-0 h1">SKA Refereeing Test (Beta)</p>
+        <p class="navbar-brand mb-0 h1">SKA Refereeing Test</p>
         <div class="quiz-console w-100" style="display: none;" id="q-topbar">
             <div class="d-flex justify-content align-middle">
                 <div class="container d-flex justify-content-center">

ref-test/app/tools/auth.py

@@ -1,10 +1,8 @@
-from .data import load
 from ..models import User
 from ..tools.logs import write
 
 from flask.helpers import abort, flash, redirect, url_for
 from flask_login import current_user
-from sqlalchemy.exc import SQLAlchemyError
 
 from functools import wraps
 
@@ -15,7 +13,7 @@ def require_account_creation(function):
             if User.query.count() == 0:
                 flash('Please register a user account.', 'alert')
                 return redirect(url_for('admin._register'))
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when checking for existing accounts: {exception}')
             return abort(500)
         return function(*args, **kwargs)

ref-test/app/tools/data.py

@@ -3,7 +3,6 @@ from ..tools.logs import write
 
 from flask import current_app as app
 from flask.helpers import abort, flash, redirect, url_for
-from sqlalchemy.exc import SQLAlchemyError
 
 import json
 from pathlib import Path
@@ -78,7 +77,7 @@ def check_dataset_exists(function):
     @wraps(function)
     def wrapper(*args, **kwargs):
         try: datasets = Dataset.query.all()
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when checking existing datasets: {exception}')
             return abort(500)
         if not datasets:

ref-test/app/tools/forms.py

@@ -1,32 +1,8 @@
-
-from ..extensions import db
 from ..tools.logs import write
 
 from flask import jsonify
-from sqlalchemy.exc import SQLAlchemyError
 from wtforms.validators import ValidationError
 
-import json
-from sqlalchemy.ext import mutable
-
-class JsonEncodedDict(db.TypeDecorator):
-    """Enables JSON storage by encoding and decoding on the fly."""
-    impl = db.Text
-
-    def process_bind_param(self, value, dialect):
-        if value is None:
-            return '{}'
-        else:
-            return json.dumps(value)
-
-    def process_result_value(self, value, dialect):
-        if value is None:
-            return {}
-        else:
-            return json.loads(value)
-
-mutable.MutableDict.associate_with(JsonEncodedDict)
-
 def value(min:int=0, max:int=None):
     if not max:
         message = f'Value must be greater than {min}.'
@@ -50,7 +26,7 @@ def get_time_options():
 def get_dataset_choices():
     from ..models import Dataset
     try: datasets = Dataset.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
         write('system.log', f'Database error when fetching dataset lists: {exception}')
         return []
     dataset_choices = []

ref-test/app/tools/test.py

@@ -4,7 +4,6 @@ from ..tools.logs import write
 
 from flask import request, session
 from flask.helpers import abort, redirect, url_for
-from sqlalchemy.exc import SQLAlchemyError
 
 from functools import wraps
 
@@ -133,7 +132,7 @@ def redirect_if_started(function):
         id = session.get('id')
         try:
             if request.method == 'GET' and id and Entry.query.filter_by(id=id).first(): return redirect(url_for('quiz._quiz'))
-        except SQLAlchemyError as exception:
+        except Exception as exception:
             write('system.log', f'Database error when checking if test has been started: {exception}')
             return abort(500)
         return function(*args, **kwargs)

ref-test/app/view/templates/view/components/navbar.html

@@ -1,6 +1,6 @@
 <nav class="navbar fixed-top navbar-expand-md navbar-dark bg-dark">
     <div class="container">
-        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest (Beta) | Admin</a>
+        <a href="{{ url_for('admin._home') }}" class="navbar-brand mb-0 h1">RefTest | Admin</a>
         <button
             class="navbar-toggler"
             type="button"

ref-test/app/view/views.py

@@ -7,7 +7,6 @@ from ..tools.logs import write
 from flask import Blueprint, jsonify, render_template, request
 from flask.helpers import abort, flash, redirect, url_for
 from flask_login import login_required
-from sqlalchemy.exc import SQLAlchemyError
 
 view = Blueprint(
     name='view',
@@ -38,7 +37,7 @@ def _view_console(id:str=None):
         dataset = Dataset.query.filter_by(id=id).first()
         datasets = Dataset.query.count()
         users = User.query.all()
-    except SQLAlchemyError as exception:
+    except Exception as exception:
             write('system.log', f'Database error when processing request \'{request.url}\': {exception}')
             return abort(500)
     if not dataset:

ref-test/install.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 from main import app
 from app.extensions import db
+from app.models import *
 from app.tools.data import save
 from app.tools.logs import write
 from sqlalchemy_utils import create_database, database_exists
@@ -21,14 +22,14 @@ with app.app_context():
     if not path.isfile(f'./{data}/logs/users.log'): write('users.log', 'Log file created.')
     if not path.isfile(f'./{data}/logs/system.log'): write('system.log', 'Log file created.')
     if not path.isfile(f'./{data}/logs/tests.log'): write('tests.log', 'Log file created.')
-    if not database_exists(database_uri):
-        create_database(database_uri)
-        write('system.log', 'No database found. Creating a new database.')
-        from app.models import *
-        db.create_all()
-        write('system.log', 'Creating database schema.')
     if not path.isfile(f'./{data}/.encryption.key'):
         write('system.log', 'No encryption key found. Generating new encryption key.')
         with open(f'./{data}/.encryption.key', 'wb') as key_file:
             key = Fernet.generate_key()
-            key_file.write(key)
+            key_file.write(key)
+
+    """Create File for SQLite Database"""
+    if database_uri[0:6].lower() == 'sqlite':
+        if not database_exists(database_uri):
+            create_database(database_uri)
+            write('system.log', 'No SQLite file found. Creating a new database.')

ref-test/requirements.txt

@@ -1,7 +1,7 @@
 blinker==1.5
 cffi==1.15.1
 click==8.1.3
-cryptography==37.0.4
+cryptography==38.0.1
 dnspython==2.2.1
 dominate==2.7.0
 email-validator==1.2.1
@@ -11,18 +11,22 @@ Flask-Login==0.6.2
 Flask-Mail==0.9.1
 Flask-SQLAlchemy==2.5.1
 Flask-WTF==1.0.1
-greenlet==1.1.2
+greenlet==1.1.3
 gunicorn==20.1.0
 idna==3.3
 itsdangerous==2.1.2
 Jinja2==3.1.2
 MarkupSafe==2.1.1
+pip==22.2.2
 pycparser==2.21
 PyMySQL==1.0.2
-python-dotenv==0.20.0
+python-dotenv==0.21.0
+setuptools==65.3.0
 six==1.16.0
-SQLAlchemy==1.4.40
+SQLAlchemy==1.4.41
+sqlalchemy-json==0.5.0
 SQLAlchemy-Utils==0.38.3
 visitor==0.1.3
 Werkzeug==2.2.2
+wheel==0.37.1
 WTForms==3.0.1

ref-test/reset.py

@@ -2,15 +2,12 @@
 from main import app
 from app.models import User
 
-from sqlalchemy.exc import SQLAlchemyError
-
 import sys
 from getpass import getpass
 
 with app.app_context():
     try: users = User.query.all()
-    except SQLAlchemyError as exception:
-        sys.exit('Database error:', exception)
+    except Exception as exception: sys.exit('Database error:', exception)
     print('')
     print('This interface will allow you to override the password for an administrator account.')
     print('To exit this interface, press Ctrl + C.')