Finished data upload

Refactored to move security package inside common
Moved data folder to process root.

REFERENCES.md

@@ -77,3 +77,7 @@ Uses SQL rather than MongoDB.
 ### Flask techniques
 
 - [Create a `config.py` file](https://www.youtube.com/watch?v=GW_2O9CrnSU)
+
+### Flask handling file uploads
+
+- [Handlin File Uploads](https://blog.miguelgrinberg.com/post/handling-file-uploads-with-flask)

ref-test/admin/auth.py

@@ -3,7 +3,7 @@ from flask.helpers import flash, url_for
 from flask.json import jsonify
 from .models.users import User
 from uuid import uuid4
-from security.database import decrypt_find_one, encrypted_update
+from common.security.database import decrypt_find_one, encrypted_update
 from werkzeug.security import check_password_hash
 
 from main import db

ref-test/admin/models/forms.py

@@ -1,4 +1,5 @@
 from flask_wtf import FlaskForm
+from flask_wtf.file import FileField, FileRequired, FileAllowed
 from wtforms import StringField, PasswordField, BooleanField, DateField, SelectField
 from wtforms.validators import InputRequired, Email, Length, EqualTo, Optional
 from datetime import date, timedelta
@@ -54,3 +55,6 @@ class CreateTest(FlaskForm):
     ]
     expiry_date = DateField('Expiry Date', format="%Y-%m-%d", validators=[InputRequired()], default = date.today() + timedelta(days=1) )
     time_limit = SelectField('Time Limit', choices=time_options)
+
+class UploadDataForm(FlaskForm):
+    data_file = FileField('Data File', validators=[FileRequired(), FileAllowed(['json'])])

ref-test/admin/models/tests.py

@@ -5,7 +5,7 @@ from flask import flash, jsonify
 import secrets
 
 from main import db
-from security import encrypt
+from common.security import encrypt
 
 class Test:
     def __init__(self, _id=None, start_date=None, expiry_date=None, time_limit=None, creator=None):

ref-test/admin/static/js/script.js

@@ -350,13 +350,18 @@ $('form[name=form-update-account]').submit(function(event) {
     event.preventDefault();
 });
 
-$('.delete-test').click(function(event) {
+$('form[name=form-create-test]').submit(function(event) {
     
-    _id = $(this).data('_id')
+    var $form = $(this);
+    var alert = document.getElementById('alert-box');
+    var data = $form.serialize();
+    alert.innerHTML = ''
 
     $.ajax({
-        url: `/admin/tests/delete/${_id}`,
-        type: 'GET',
+        url: window.location.pathname,
+        type: 'POST',
+        data: data,
+        dataType: 'json',
         success: function(response) {
             window.location.href = '/admin/tests/';
         },
@@ -386,20 +391,78 @@ $('.delete-test').click(function(event) {
     event.preventDefault();
 });
 
-// Edit and Delete Test Button Handlers
-
-$('form[name=form-create-test]').submit(function(event) {
+$('form[name=form-upload-questions]').submit(function(event) {
     
     var $form = $(this);
     var alert = document.getElementById('alert-box');
-    var data = $form.serialize();
+    var data = new FormData($form[0]);
+    var file = $('input[name=data_file]')[0].files[0]
+    data.append('file', file)
     alert.innerHTML = ''
 
     $.ajax({
         url: window.location.pathname,
         type: 'POST',
         data: data,
-        dataType: 'json',
+        processData: false,
+        contentType: false,
+        success: function(response) {
+            if (typeof response.success === 'string' || response.success instanceof String) {
+                alert.innerHTML = alert.innerHTML + `
+                <div class="alert alert-success alert-dismissible fade show" role="alert">
+                    <i class="bi bi-exclamation-triangle-fill" title="Danger"></i>
+                    ${response.success}
+                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+                </div>
+                `;
+            } else if (response.success instanceof Array) {
+                for (var i = 0; i < response.success.length; i ++) {
+                    alert.innerHTML = alert.innerHTML + `
+                    <div class="alert alert-success alert-dismissible fade show" role="alert">
+                        <i class="bi bi-exclamation-triangle-fill" title="Danger"></i>
+                        ${response.success[i]}
+                        <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+                    </div>
+                    `;
+                }
+            }
+        },
+        error: function(response) {
+            if (typeof response.responseJSON.error === 'string' || response.responseJSON.error instanceof String) {
+                alert.innerHTML = alert.innerHTML + `
+                <div class="alert alert-danger alert-dismissible fade show" role="alert">
+                    <i class="bi bi-exclamation-triangle-fill" title="Danger"></i>
+                    ${response.responseJSON.error}
+                    <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+                </div>
+                `;
+            } else if (response.responseJSON.error instanceof Array) {
+                for (var i = 0; i < response.responseJSON.error.length; i ++) {
+                    alert.innerHTML = alert.innerHTML + `
+                    <div class="alert alert-danger alert-dismissible fade show" role="alert">
+                        <i class="bi bi-exclamation-triangle-fill" title="Danger"></i>
+                        ${response.responseJSON.error[i]}
+                        <button type="button" class="btn-close" data-bs-dismiss="alert" aria-label="Close"></button>
+                    </div>
+                    `;
+                }
+            }
+        }
+    });
+
+    event.preventDefault();
+});
+
+
+// Edit and Delete Test Button Handlers
+
+$('.delete-test').click(function(event) {
+    
+    _id = $(this).data('_id')
+
+    $.ajax({
+        url: `/admin/tests/delete/${_id}`,
+        type: 'GET',
         success: function(response) {
             window.location.href = '/admin/tests/';
         },

ref-test/admin/templates/admin/settings/questions.html

@@ -1 +1,23 @@
 {% extends "admin/components/base.html" %}
+{% block title %} SKA Referee Test | Upload Questions {% endblock %}
+{% block content %}
+    <!-- <h1>Upload Question Dataset</h1> -->
+    <div class="form-container">
+        <form name="form-upload-questions" method="post" action="#" class="form-signin" enctype="multipart/form-data">
+            <h2 class="form-signin-heading">Upload Question Dataset</h2>
+            {{ form.hidden_tag() }}
+            {{ form.data_file() }}
+            {% include "admin/components/client-alerts.html" %}
+            <div class="container form-submission-button">
+                <div class="row">
+                    <div class="col text-center">
+                        <button title="Create User" class="btn btn-md btn-success btn-block" type="submit">
+                            <i class="bi bi-file-earmark-arrow-up-fill button-icon"></i>
+                            Upload Dataset
+                        </button>
+                    </div>
+                </div>
+            </div>
+        </form>
+    </div>
+{% endblock %}

ref-test/admin/views.py

@@ -1,9 +1,10 @@
 from flask import Blueprint, render_template, flash, redirect, request, jsonify, abort
 from flask.helpers import url_for
 from functools import wraps
+from datetime import datetime
 
 from werkzeug.security import check_password_hash
-from security.database import decrypt_find, decrypt_find_one
+from common.security.database import decrypt_find, decrypt_find_one
 from .models.users import User
 from flask_mail import Message
 from main import db
@@ -231,11 +232,27 @@ def update_user(_id:str):
             errors = [*form.user_password.errors, *form.email.errors, *form.password.errors, *form.password_reenter.errors]
             return jsonify({ 'error': errors}), 400
 
-@views.route('/settings/questions/')
+@views.route('/settings/questions/', methods=['GET', 'POST'])
 @admin_account_required
 @login_required
 def questions():
-    return render_template('/admin/settings/questions.html')
+    from main import app
+    from .models.forms import UploadDataForm
+    from common.data_tools import check_json_format, validate_json_contents, store_data_file
+    form = UploadDataForm()
+    if request.method == 'GET':
+        return render_template('/admin/settings/questions.html', form=form)
+    if request.method == 'POST':
+        if form.validate_on_submit():
+            upload = form.data_file.data
+            if not check_json_format(upload):
+                return jsonify({ 'error': 'Invalid file selected. Please upload a JSON file.'}), 400
+            if not validate_json_contents(upload):
+                return jsonify({'error': 'The data in the file is invalid.'}), 400
+            store_data_file(upload)
+            return jsonify({ 'success': 'File uploaded.'}), 200
+        errors = [*form.errors]
+        return jsonify({ 'error': errors}), 400
 
 @views.route('/settings/questions/upload/')
 @admin_account_required

ref-test/common/__init__.py

@@ -1,18 +0,0 @@
-from datetime import datetime, timedelta
-from flask import Blueprint, redirect, request
-
-cookie_consent = Blueprint(
-    'cookie_consent',
-    __name__
-)
-@cookie_consent.route('/')
-def _cookies():
-    resp = redirect('/')
-    resp.set_cookie(
-        key = 'cookie_consent',
-        value = 'True',
-        max_age = timedelta(days=14) if request.cookies.get('remember') == 'True' else 'Session',
-        path = '/',
-        expires = datetime.utcnow() + timedelta(days=14) if request.cookies.get('remember') else 'Session'
-    )
-    return resp

ref-test/common/blueprints.py

@@ -0,0 +1,18 @@
+from datetime import datetime, timedelta
+from flask import Blueprint, redirect, request
+
+cookie_consent = Blueprint(
+    'cookie_consent',
+    __name__
+)
+@cookie_consent.route('/')
+def _cookies():
+    resp = redirect('/')
+    resp.set_cookie(
+        key = 'cookie_consent',
+        value = 'True',
+        max_age = timedelta(days=14) if request.cookies.get('remember') == 'True' else 'Session',
+        path = '/',
+        expires = datetime.utcnow() + timedelta(days=14) if request.cookies.get('remember') else 'Session'
+    )
+    return resp

ref-test/common/data_tools.py

@@ -0,0 +1,59 @@
+import os
+from shutil import rmtree
+import pathlib
+from json import dump, loads
+from datetime import datetime
+from main import app
+
+from werkzeug.utils import secure_filename
+
+def check_data_folder_exists():
+    if not os.path.exists(app.config['DATA_FILE_DIRECTORY']):
+        pathlib.Path(app.config['DATA_FILE_DIRECTORY']).mkdir(parents='True', exist_ok='True')
+
+def check_current_indicator():
+    if not os.path.isfile(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt')):
+        open(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt'),'w').close()
+
+def make_temp_dir(file):
+    if not os.path.isdir('tmp'):
+        os.mkdir('tmp')
+    if os.path.isfile(f'tmp/{file.filename}'):
+        os.remove(f'tmp/{file.filename}')
+    file.save(f'tmp/{file.filename}')
+
+def check_json_format(file):
+    if not '.' in file.filename:
+        return False
+    if not file.filename.rsplit('.', 1)[-1] == 'json':
+        return False
+    return True
+
+def validate_json_contents(file):
+    file.stream.seek(0)
+    data = loads(file.read())
+    if not type(data) is dict:
+        return False
+    elif not all( key in data for key in ['meta', 'questions']):
+        return False
+    elif not type(data['meta']) is dict:
+        return False
+    elif not type(data['questions']) is list:
+        return False
+    return True
+
+def store_data_file(file):
+    from admin.views import get_id_from_cookie
+    check_current_indicator()
+    timestamp = datetime.utcnow()
+    filename = '.'.join([timestamp.strftime('%Y%m%d%H%M%S'),'json'])
+    filename = secure_filename(filename)
+    file_path = os.path.join(app.config['DATA_FILE_DIRECTORY'], filename)
+    file.stream.seek(0)
+    data = loads(file.read())
+    data['meta']['timestamp'] = timestamp.strftime('%Y-%m-%d %H%M%S')
+    data['meta']['author'] = get_id_from_cookie()
+    with open(file_path, 'w') as _file:
+        dump(data, _file, indent=4)
+    with open(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt'), 'w') as _file:
+        _file.write(filename)

ref-test/common/security/__init__.py

@@ -2,17 +2,17 @@ from os import environ, path
 from cryptography.fernet import Fernet
 
 def generate_keyfile():
-    with open('./security/.encryption.key', 'wb') as keyfile:
+    with open('./common/security/.encryption.key', 'wb') as keyfile:
         key = Fernet.generate_key()
         keyfile.write(key)
 
 def load_key():
-    with open('./security/.encryption.key', 'rb') as keyfile:
+    with open('./common/security/.encryption.key', 'rb') as keyfile:
         key = keyfile.read()
         return key
 
 def check_keyfile_exists():
-    return path.isfile('./security/.encryption.key')
+    return path.isfile('./common/security/.encryption.key')
 
 def encrypt(input):
     if not check_keyfile_exists():

ref-test/config.py

@@ -26,6 +26,7 @@ class Config(object):
     MAIL_MAX_EMAILS = int(os.getenv("MAIL_MAX_EMAILS"))
     MAIL_SUPPRESS_SEND = False
     MAIL_ASCII_ATTACHMENTS = bool(os.getenv("MAIL_ASCII_ATTACHMENTS"))
+    DATA_FILE_DIRECTORY = os.getenv("DATA_FILE_DIRECTORY")
 
 class ProductionConfig(Config):
     pass

ref-test/data/.gitignore

@@ -0,0 +1,2 @@
+*
+!.gitignore

ref-test/main.py

@@ -9,7 +9,7 @@ from pymongo.errors import ConnectionFailure
 from flask_wtf.csrf import CSRFProtect, CSRFError
 from flask_mail import Mail
 
-from security import check_keyfile_exists, generate_keyfile
+from common.security import check_keyfile_exists, generate_keyfile
 
 app = Flask(__name__)
 app.config.from_object('config.DevelopmentConfig')
@@ -37,7 +37,7 @@ if __name__ == '__main__':
     if not check_keyfile_exists():
         generate_keyfile()
 
-    from common import cookie_consent
+    from common.blueprints import cookie_consent
 
     from admin.views import views as admin_views
     from admin.auth import auth as admin_auth

ref-test/quiz/views.py

@@ -3,7 +3,7 @@ from datetime import datetime
 from uuid import uuid4
 
 from main import db
-from security import encrypt
+from common.security import encrypt
 
 views = Blueprint(
     'quiz_views',