From c7252d0f7b5bb2b9a80e9a6edad59ed7eb9044e4 Mon Sep 17 00:00:00 2001 From: viveksantayana Date: Sun, 28 Nov 2021 02:30:46 +0000 Subject: [PATCH] Finished data upload Refactored to move security package inside common Moved data folder to process root. --- REFERENCES.md | 4 + ref-test/admin/auth.py | 2 +- ref-test/admin/models/forms.py | 6 +- ref-test/admin/models/tests.py | 2 +- ref-test/admin/models/users.py | 4 +- ref-test/admin/static/js/script.js | 81 ++++++++++++++++--- .../templates/admin/settings/questions.html | 24 +++++- ref-test/admin/views.py | 23 +++++- ref-test/common/__init__.py | 18 ----- ref-test/common/blueprints.py | 18 +++++ ref-test/common/data_tools.py | 59 ++++++++++++++ ref-test/{ => common}/security/__init__.py | 6 +- ref-test/{ => common}/security/database.py | 0 ref-test/config.py | 1 + ref-test/data/.gitignore | 2 + ref-test/main.py | 4 +- ref-test/quiz/views.py | 2 +- 17 files changed, 214 insertions(+), 42 deletions(-) create mode 100644 ref-test/common/blueprints.py create mode 100644 ref-test/common/data_tools.py rename ref-test/{ => common}/security/__init__.py (86%) rename ref-test/{ => common}/security/database.py (100%) create mode 100644 ref-test/data/.gitignore diff --git a/REFERENCES.md b/REFERENCES.md index 1f58c48..56aed1c 100644 --- a/REFERENCES.md +++ b/REFERENCES.md @@ -77,3 +77,7 @@ Uses SQL rather than MongoDB. ### Flask techniques - [Create a `config.py` file](https://www.youtube.com/watch?v=GW_2O9CrnSU) + +### Flask handling file uploads + +- [Handlin File Uploads](https://blog.miguelgrinberg.com/post/handling-file-uploads-with-flask) \ No newline at end of file diff --git a/ref-test/admin/auth.py b/ref-test/admin/auth.py index 2c6e259..7950c99 100644 --- a/ref-test/admin/auth.py +++ b/ref-test/admin/auth.py @@ -3,7 +3,7 @@ from flask.helpers import flash, url_for from flask.json import jsonify from .models.users import User from uuid import uuid4 -from security.database import decrypt_find_one, encrypted_update +from common.security.database import decrypt_find_one, encrypted_update from werkzeug.security import check_password_hash from main import db diff --git a/ref-test/admin/models/forms.py b/ref-test/admin/models/forms.py index c63b8cb..9e762f9 100644 --- a/ref-test/admin/models/forms.py +++ b/ref-test/admin/models/forms.py @@ -1,4 +1,5 @@ from flask_wtf import FlaskForm +from flask_wtf.file import FileField, FileRequired, FileAllowed from wtforms import StringField, PasswordField, BooleanField, DateField, SelectField from wtforms.validators import InputRequired, Email, Length, EqualTo, Optional from datetime import date, timedelta @@ -53,4 +54,7 @@ class CreateTest(FlaskForm): ('120', '2 hours') ] expiry_date = DateField('Expiry Date', format="%Y-%m-%d", validators=[InputRequired()], default = date.today() + timedelta(days=1) ) - time_limit = SelectField('Time Limit', choices=time_options) \ No newline at end of file + time_limit = SelectField('Time Limit', choices=time_options) + +class UploadDataForm(FlaskForm): + data_file = FileField('Data File', validators=[FileRequired(), FileAllowed(['json'])]) \ No newline at end of file diff --git a/ref-test/admin/models/tests.py b/ref-test/admin/models/tests.py index ce26428..45196a7 100644 --- a/ref-test/admin/models/tests.py +++ b/ref-test/admin/models/tests.py @@ -5,7 +5,7 @@ from flask import flash, jsonify import secrets from main import db -from security import encrypt +from common.security import encrypt class Test: def __init__(self, _id=None, start_date=None, expiry_date=None, time_limit=None, creator=None): diff --git a/ref-test/admin/models/users.py b/ref-test/admin/models/users.py index a983975..5db7156 100644 --- a/ref-test/admin/models/users.py +++ b/ref-test/admin/models/users.py @@ -6,8 +6,8 @@ from werkzeug.utils import redirect from flask_mail import Message import secrets -from security import encrypt, decrypt -from security.database import decrypt_find_one, encrypted_update +from common.security import encrypt, decrypt +from common.security.database import decrypt_find_one, encrypted_update from datetime import datetime, timedelta from main import db, mail diff --git a/ref-test/admin/static/js/script.js b/ref-test/admin/static/js/script.js index ad97048..e6e4d73 100644 --- a/ref-test/admin/static/js/script.js +++ b/ref-test/admin/static/js/script.js @@ -350,13 +350,18 @@ $('form[name=form-update-account]').submit(function(event) { event.preventDefault(); }); -$('.delete-test').click(function(event) { +$('form[name=form-create-test]').submit(function(event) { - _id = $(this).data('_id') + var $form = $(this); + var alert = document.getElementById('alert-box'); + var data = $form.serialize(); + alert.innerHTML = '' $.ajax({ - url: `/admin/tests/delete/${_id}`, - type: 'GET', + url: window.location.pathname, + type: 'POST', + data: data, + dataType: 'json', success: function(response) { window.location.href = '/admin/tests/'; }, @@ -386,20 +391,78 @@ $('.delete-test').click(function(event) { event.preventDefault(); }); -// Edit and Delete Test Button Handlers - -$('form[name=form-create-test]').submit(function(event) { +$('form[name=form-upload-questions]').submit(function(event) { var $form = $(this); var alert = document.getElementById('alert-box'); - var data = $form.serialize(); + var data = new FormData($form[0]); + var file = $('input[name=data_file]')[0].files[0] + data.append('file', file) alert.innerHTML = '' $.ajax({ url: window.location.pathname, type: 'POST', data: data, - dataType: 'json', + processData: false, + contentType: false, + success: function(response) { + if (typeof response.success === 'string' || response.success instanceof String) { + alert.innerHTML = alert.innerHTML + ` + + `; + } else if (response.success instanceof Array) { + for (var i = 0; i < response.success.length; i ++) { + alert.innerHTML = alert.innerHTML + ` + + `; + } + } + }, + error: function(response) { + if (typeof response.responseJSON.error === 'string' || response.responseJSON.error instanceof String) { + alert.innerHTML = alert.innerHTML + ` + + `; + } else if (response.responseJSON.error instanceof Array) { + for (var i = 0; i < response.responseJSON.error.length; i ++) { + alert.innerHTML = alert.innerHTML + ` + + `; + } + } + } + }); + + event.preventDefault(); +}); + + +// Edit and Delete Test Button Handlers + +$('.delete-test').click(function(event) { + + _id = $(this).data('_id') + + $.ajax({ + url: `/admin/tests/delete/${_id}`, + type: 'GET', success: function(response) { window.location.href = '/admin/tests/'; }, diff --git a/ref-test/admin/templates/admin/settings/questions.html b/ref-test/admin/templates/admin/settings/questions.html index 9624223..d4892e3 100644 --- a/ref-test/admin/templates/admin/settings/questions.html +++ b/ref-test/admin/templates/admin/settings/questions.html @@ -1 +1,23 @@ -{% extends "admin/components/base.html" %} \ No newline at end of file +{% extends "admin/components/base.html" %} +{% block title %} SKA Referee Test | Upload Questions {% endblock %} +{% block content %} + +
+ +
+{% endblock %} \ No newline at end of file diff --git a/ref-test/admin/views.py b/ref-test/admin/views.py index 6bc4ce4..43edb22 100644 --- a/ref-test/admin/views.py +++ b/ref-test/admin/views.py @@ -1,9 +1,10 @@ from flask import Blueprint, render_template, flash, redirect, request, jsonify, abort from flask.helpers import url_for from functools import wraps +from datetime import datetime from werkzeug.security import check_password_hash -from security.database import decrypt_find, decrypt_find_one +from common.security.database import decrypt_find, decrypt_find_one from .models.users import User from flask_mail import Message from main import db @@ -231,11 +232,27 @@ def update_user(_id:str): errors = [*form.user_password.errors, *form.email.errors, *form.password.errors, *form.password_reenter.errors] return jsonify({ 'error': errors}), 400 -@views.route('/settings/questions/') +@views.route('/settings/questions/', methods=['GET', 'POST']) @admin_account_required @login_required def questions(): - return render_template('/admin/settings/questions.html') + from main import app + from .models.forms import UploadDataForm + from common.data_tools import check_json_format, validate_json_contents, store_data_file + form = UploadDataForm() + if request.method == 'GET': + return render_template('/admin/settings/questions.html', form=form) + if request.method == 'POST': + if form.validate_on_submit(): + upload = form.data_file.data + if not check_json_format(upload): + return jsonify({ 'error': 'Invalid file selected. Please upload a JSON file.'}), 400 + if not validate_json_contents(upload): + return jsonify({'error': 'The data in the file is invalid.'}), 400 + store_data_file(upload) + return jsonify({ 'success': 'File uploaded.'}), 200 + errors = [*form.errors] + return jsonify({ 'error': errors}), 400 @views.route('/settings/questions/upload/') @admin_account_required diff --git a/ref-test/common/__init__.py b/ref-test/common/__init__.py index d15e272..e69de29 100644 --- a/ref-test/common/__init__.py +++ b/ref-test/common/__init__.py @@ -1,18 +0,0 @@ -from datetime import datetime, timedelta -from flask import Blueprint, redirect, request - -cookie_consent = Blueprint( - 'cookie_consent', - __name__ -) -@cookie_consent.route('/') -def _cookies(): - resp = redirect('/') - resp.set_cookie( - key = 'cookie_consent', - value = 'True', - max_age = timedelta(days=14) if request.cookies.get('remember') == 'True' else 'Session', - path = '/', - expires = datetime.utcnow() + timedelta(days=14) if request.cookies.get('remember') else 'Session' - ) - return resp \ No newline at end of file diff --git a/ref-test/common/blueprints.py b/ref-test/common/blueprints.py new file mode 100644 index 0000000..d15e272 --- /dev/null +++ b/ref-test/common/blueprints.py @@ -0,0 +1,18 @@ +from datetime import datetime, timedelta +from flask import Blueprint, redirect, request + +cookie_consent = Blueprint( + 'cookie_consent', + __name__ +) +@cookie_consent.route('/') +def _cookies(): + resp = redirect('/') + resp.set_cookie( + key = 'cookie_consent', + value = 'True', + max_age = timedelta(days=14) if request.cookies.get('remember') == 'True' else 'Session', + path = '/', + expires = datetime.utcnow() + timedelta(days=14) if request.cookies.get('remember') else 'Session' + ) + return resp \ No newline at end of file diff --git a/ref-test/common/data_tools.py b/ref-test/common/data_tools.py new file mode 100644 index 0000000..dbc92e8 --- /dev/null +++ b/ref-test/common/data_tools.py @@ -0,0 +1,59 @@ +import os +from shutil import rmtree +import pathlib +from json import dump, loads +from datetime import datetime +from main import app + +from werkzeug.utils import secure_filename + +def check_data_folder_exists(): + if not os.path.exists(app.config['DATA_FILE_DIRECTORY']): + pathlib.Path(app.config['DATA_FILE_DIRECTORY']).mkdir(parents='True', exist_ok='True') + +def check_current_indicator(): + if not os.path.isfile(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt')): + open(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt'),'w').close() + +def make_temp_dir(file): + if not os.path.isdir('tmp'): + os.mkdir('tmp') + if os.path.isfile(f'tmp/{file.filename}'): + os.remove(f'tmp/{file.filename}') + file.save(f'tmp/{file.filename}') + +def check_json_format(file): + if not '.' in file.filename: + return False + if not file.filename.rsplit('.', 1)[-1] == 'json': + return False + return True + +def validate_json_contents(file): + file.stream.seek(0) + data = loads(file.read()) + if not type(data) is dict: + return False + elif not all( key in data for key in ['meta', 'questions']): + return False + elif not type(data['meta']) is dict: + return False + elif not type(data['questions']) is list: + return False + return True + +def store_data_file(file): + from admin.views import get_id_from_cookie + check_current_indicator() + timestamp = datetime.utcnow() + filename = '.'.join([timestamp.strftime('%Y%m%d%H%M%S'),'json']) + filename = secure_filename(filename) + file_path = os.path.join(app.config['DATA_FILE_DIRECTORY'], filename) + file.stream.seek(0) + data = loads(file.read()) + data['meta']['timestamp'] = timestamp.strftime('%Y-%m-%d %H%M%S') + data['meta']['author'] = get_id_from_cookie() + with open(file_path, 'w') as _file: + dump(data, _file, indent=4) + with open(os.path.join(app.config['DATA_FILE_DIRECTORY'], '.current.txt'), 'w') as _file: + _file.write(filename) \ No newline at end of file diff --git a/ref-test/security/__init__.py b/ref-test/common/security/__init__.py similarity index 86% rename from ref-test/security/__init__.py rename to ref-test/common/security/__init__.py index 83d22c2..0272741 100644 --- a/ref-test/security/__init__.py +++ b/ref-test/common/security/__init__.py @@ -2,17 +2,17 @@ from os import environ, path from cryptography.fernet import Fernet def generate_keyfile(): - with open('./security/.encryption.key', 'wb') as keyfile: + with open('./common/security/.encryption.key', 'wb') as keyfile: key = Fernet.generate_key() keyfile.write(key) def load_key(): - with open('./security/.encryption.key', 'rb') as keyfile: + with open('./common/security/.encryption.key', 'rb') as keyfile: key = keyfile.read() return key def check_keyfile_exists(): - return path.isfile('./security/.encryption.key') + return path.isfile('./common/security/.encryption.key') def encrypt(input): if not check_keyfile_exists(): diff --git a/ref-test/security/database.py b/ref-test/common/security/database.py similarity index 100% rename from ref-test/security/database.py rename to ref-test/common/security/database.py diff --git a/ref-test/config.py b/ref-test/config.py index d394ce9..a1012c5 100644 --- a/ref-test/config.py +++ b/ref-test/config.py @@ -26,6 +26,7 @@ class Config(object): MAIL_MAX_EMAILS = int(os.getenv("MAIL_MAX_EMAILS")) MAIL_SUPPRESS_SEND = False MAIL_ASCII_ATTACHMENTS = bool(os.getenv("MAIL_ASCII_ATTACHMENTS")) + DATA_FILE_DIRECTORY = os.getenv("DATA_FILE_DIRECTORY") class ProductionConfig(Config): pass diff --git a/ref-test/data/.gitignore b/ref-test/data/.gitignore new file mode 100644 index 0000000..c96a04f --- /dev/null +++ b/ref-test/data/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/ref-test/main.py b/ref-test/main.py index 2a3173c..d2f4eb3 100644 --- a/ref-test/main.py +++ b/ref-test/main.py @@ -9,7 +9,7 @@ from pymongo.errors import ConnectionFailure from flask_wtf.csrf import CSRFProtect, CSRFError from flask_mail import Mail -from security import check_keyfile_exists, generate_keyfile +from common.security import check_keyfile_exists, generate_keyfile app = Flask(__name__) app.config.from_object('config.DevelopmentConfig') @@ -37,7 +37,7 @@ if __name__ == '__main__': if not check_keyfile_exists(): generate_keyfile() - from common import cookie_consent + from common.blueprints import cookie_consent from admin.views import views as admin_views from admin.auth import auth as admin_auth diff --git a/ref-test/quiz/views.py b/ref-test/quiz/views.py index 3f0bb35..cd88bd2 100644 --- a/ref-test/quiz/views.py +++ b/ref-test/quiz/views.py @@ -3,7 +3,7 @@ from datetime import datetime from uuid import uuid4 from main import db -from security import encrypt +from common.security import encrypt views = Blueprint( 'quiz_views',