viveksantayana/ska-referee-test
1
0
Fork 0
Code Issues 1 Pull Requests Projects Releases 19 Wiki Activity

Finished quiz and debugging

Browse Source
This commit is contained in:
Vivek Santayana
2022-06-16 10:44:48 +01:00
parent b9d45f94fe
commit 4b08c830a1
15 changed files with 148 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
ref-test/app/admin/templates/admin/components/certificate.html
View File

@@ -15,30 +15,30 @@
<h5 class="mb-1">Candidate</h5>
</div>
<h2>
{{ entry.name.surname}}, {{ entry.name.first_name }}
{{ entry.get_surname()}}, {{ entry.get_first_name() }}
</h2>
</li>
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Email Address</h5>
</div>
{{ entry.email }}
{{ entry.get_email() }}
</li>
{% if entry['club'] %}
{% if entry.club %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Club</h5>
</div>
{{ entry.club }}
{{ entry.get_club() }}
</li>
{% endif %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Exam Code</h5>
</div>
{{ '—'.join([entry.test_code[:4], entry.test_code[4:8], entry.test_code[8:]]) }}
{{ entry.test.get_code() }}
</li>
{% if entry['user_code'] %}
{% if entry.user_code %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">User Code</h5>
@@ -59,19 +59,19 @@
<span class="badge bg-danger">Late</span>
{% endif %}
</div>
{{ entry.submission_time.strftime('%d %b %Y %H:%M:%S') }}
{{ entry.end_time.strftime('%d %b %Y %H:%M:%S') }}
</li>
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Score</h5>
</div>
{{ entry.results.score }}&percnt;
{{ entry.result.score }}&percnt;
</li>
<li class="list-group-item list-group-item-action {% if entry.results.grade == 'fail' %}list-group-item-danger {% elif entry.results.grade == 'merit' %} list-group-item-success {% endif %}">
<li class="list-group-item list-group-item-action {% if entry.result.grade == 'fail' %}list-group-item-danger {% elif entry.result.grade == 'merit' %} list-group-item-success {% endif %}">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Grade</h5>
</div>
{{ entry.results.grade[0]|upper }}{{ entry.results.grade[1:]}}
{{ entry.result.grade[0]|upper }}{{ entry.result.grade[1:]}}
</li>
</ul>
<div class="site-footer mt-5">

35
ref-test/app/admin/templates/admin/components/navbar.html
View File

@@ -23,8 +23,37 @@
<li class="nav-item" id="nav-results">
<a href="{{ url_for('admin._view_entries') }}" id="link-results" class="nav-link">View Results</a>
</li>
<li class="nav-item" id="nav-tests">
<a href="{{ url_for('admin._tests') }}" id="link-tests" class="nav-link">Manage Exams</a>
<li class="nav-item dropdown" id="nav-tests">
<a
class="nav-link dropdown-toggle"
id="dropdown-tests"
role="button"
href="{{ url_for('admin._tests') }}"
data-bs-toggle="dropdown"
aria-expanded="false"
>
Exams
</a>
<ul
class="dropdown-menu"
aria-labelledby="dropdown-settings"
>
<li>
<a href="{{ url_for('admin._tests', filter='active') }}" id="link-active" class="dropdown-item">Active</a>
</li>
<li>
<a href="{{ url_for('admin._tests', filter='scheduled') }}" id="link-scheduled" class="dropdown-item">Scheduled</a>
</li>
<li>
<a href="{{ url_for('admin._tests', filter='expired') }}" id="link-expired" class="dropdown-item">Expired</a>
</li>
<li>
<a href="{{ url_for('admin._tests', filter='all') }}" id="link-all" class="dropdown-item">All</a>
</li>
<li>
<a href="{{ url_for('admin._tests', filter='create') }}" id="link-create" class="dropdown-item">Create</a>
</li>
</ul>
</li>
<li class="nav-item dropdown" id="nav-settings">
<a
@@ -42,7 +71,7 @@
aria-labelledby="dropdown-settings"
>
<li>
<a href="{{ url_for('admin._settings') }}" id="link-settings" class="dropdown-item">Settings</a>
<a href="{{ url_for('admin._settings') }}" id="link-settings" class="dropdown-item">View Settings</a>
</li>
<li>
<a href="{{ url_for('admin._users') }}" id="link-users" class="dropdown-item">Users</a>

2
ref-test/app/admin/templates/admin/components/secondary-navs/tests.html
View File

@@ -1,7 +1,7 @@
<div class="navbar navbar-expand-sm navbar-light bg-light">
<div class="container-fluid">
<div class="expand navbar-expand justify-content-center" id="navbar_secondary">
<ul class="nav">
<ul class="nav nav-pills">
<li class="nav-item">
<a class="nav-link" href="{{ url_for('admin._tests', filter='active') }}">Active</a>
</li>

6
ref-test/app/admin/templates/admin/index.html
View File

@@ -69,13 +69,13 @@
{% for result in recent_results %}
<tr>
<td>
<a href="{{ url_for('admin._view_entry', id=result.id) }}">{{ result.name.surname }}, {{ result.name.first_name }}</a>
<a href="{{ url_for('admin._view_entry', id=result.id) }}">{{ result.get_surname() }}, {{ result.get_first_name() }}</a>
</td>
<td>
{{ result.submission_time.strftime('%d %b %Y %H:%M') }}
{{ result.end_time.strftime('%d %b %Y %H:%M') }}
</td>
<td>
{{ result.percent }}&percnt; ({{ result.results.grade }})
{{ (100*result.result['score']/result.result['max'])|round|int }}&percnt; ({{ result.result.grade }})
</td>
</tr>
{% endfor %}

34
ref-test/app/admin/templates/admin/result-detail.html
View File

@@ -13,30 +13,30 @@
<h5 class="mb-1">Candidate</h5>
</div>
<h2>
{{ entry.name.surname }}, {{ entry.name.first_name }}
{{ entry.get_surname() }}, {{ entry.get_first_name() }}
</h2>
</li>
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Email Address</h5>
</div>
{{ entry.email }}
{{ entry.get_email() }}
</li>
{% if entry['club'] %}
{% if entry.club %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Club</h5>
</div>
{{ entry.club }}
{{ entry.get_club() }}
</li>
{% endif %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Exam Code</h5>
</div>
{{ '—'.join([entry.test_code[:4], entry.test_code[4:8], entry.test_code[8:]]) }}
{{ entry.test.get_code() }}
</li>
{% if entry['user_code'] %}
{% if entry.user_code %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">User Code</h5>
@@ -44,7 +44,7 @@
{{ entry.user_code }}
</li>
{% endif %}
{% if 'start_time' in entry %}
{% if entry.start_time %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Start Time</h5>
@@ -59,28 +59,28 @@
<span class="badge bg-danger">Late</span>
{% endif %}
</div>
{% if 'submission_time' in entry %}
{{ entry.submission_time.strftime('%d %b %Y %H:%M:%S') }}
{% if entry.end_time %}
{{ entry.end_time.strftime('%d %b %Y %H:%M:%S') }}
{% else %}
Incomplete
{% endif %}
</li>
{% if 'results' in entry %}
{% if entry.result %}
<li class="list-group-item list-group-item-action">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Score</h5>
</div>
{{ entry.results.score }}&percnt;
{{ entry.result.score }}&percnt;
</li>
<li class="list-group-item list-group-item-action {% if entry.results.grade == 'fail' %}list-group-item-danger {% elif entry.results.grade == 'merit' %} list-group-item-success {% endif %}">
<li class="list-group-item list-group-item-action {% if entry.result.grade == 'fail' %}list-group-item-danger {% elif entry.result.grade == 'merit' %} list-group-item-success {% endif %}">
<div class="d-flex w-100 justify-content-between">
<h5 class="mb-1">Grade</h5>
</div>
{{ entry.results.grade[0]|upper }}{{ entry.results.grade[1:]}}
{{ entry.result.grade[0]|upper }}{{ entry.result.grade[1:]}}
</li>
{% endif %}
</ul>
{% if 'results' in entry %}
{% if entry.result %}
<div class="accordion" id="results-breakdown">
<div class="accordion-item">
<h2 class="accordion-header" id="by-category">
@@ -105,7 +105,7 @@
</tr>
</thead>
<tbody>
{% for tag, scores in entry.results.tags.items() %}
{% for tag, scores in entry.result.tags.items() %}
<tr>
<td>
{{ tag }}
@@ -149,8 +149,8 @@
{{ question }}
</td>
<td>
{{ answer }}
{% if not correct[question] == answer %}
{{ answers[question|int][answer|int] }}
{% if not correct[question] == answer|int %}
<span class="badge badge-pill bg-danger badge-danger">Incorrect</span>
{% endif %}
</td>

22
ref-test/app/admin/templates/admin/results.html
View File

@@ -37,34 +37,34 @@
{% for entry in entries %}
<tr class="table-row">
<td>
{{ entry.name.surname }}, {{ entry.name.first_name }}
{{ entry.get_surname() }}, {{ entry.get_first_name() }}
</td>
<td>
{% if 'club' in entry %}
{{ entry.club }}
{% if entry.club %}
{{ entry.get_club() }}
{% endif %}
</td>
<td>
{{ '—'.join([entry.test_code[:4], entry.test_code[4:8], entry.test_code[8:]]) }}
{{ entry.test.get_code() }}
</td>
<td>
{% if 'status' in entry %}
{% if entry.status %}
{{ entry.status }}
{% endif %}
</td>
<td>
{% if 'submission_time' in entry %}
{{ entry.submission_time.strftime('%d %b %Y') }}
{% if entry.end_time %}
{{ entry.end_time.strftime('%d %b %Y') }}
{% endif %}
</td>
<td>
{% if 'results' in entry %}
{{ entry.results.score }}&percnt;
{% if entry.result %}
{{ entry.result.score }}&percnt;
{% endif %}
</td>
<td>
{% if 'results' in entry %}
{{ entry.results.grade }}
{% if entry.result %}
{{ entry.result.grade }}
{% endif %}
</td>
<td class="row-actions">

15
ref-test/app/admin/views.py
View File

@@ -3,7 +3,7 @@ from ..models import Dataset, Entry, Test, User
from ..tools.auth import disable_if_logged_in, require_account_creation
from ..tools.forms import get_dataset_choices, get_time_options
from ..tools.data import check_is_json, validate_json
from ..tools.test import get_correct_answers
from ..tools.test import answer_options, get_correct_answers
from flask import Blueprint, jsonify, render_template, redirect, request, session
from flask.helpers import flash, url_for
@@ -33,8 +33,6 @@ def _home():
upcoming_tests.sort(key= lambda x: x.start_date)
recent_results = [result for result in results if not result.status == 'started' ]
recent_results.sort(key= lambda x: x.end_time, reverse=True)
for result in recent_results:
result['percent'] = round(100*result['result']['score']/result['result']['max'])
return render_template('/admin/index.html', current_tests = current_tests, upcomimg_tests = upcoming_tests, recent_results = recent_results)
@admin.route('/settings/')
@@ -255,7 +253,7 @@ def _tests(filter:str=None):
if not datasets:
flash('There are no available question datasets. Please upload a question dataset in order to set up an exam.', 'error')
return redirect(url_for('admin._questions'))
if filter not in [None, '', 'create','active','scheduled','expired','all']: return redirect(url_for('admin._tests'))
if filter not in ['create','active','scheduled','expired','all']: return redirect(url_for('admin._tests', filter='active'))
if filter == 'create':
form = CreateTest()
form.time_limit.choices = get_time_options()
@@ -337,7 +335,7 @@ def _view_test(id:str=None):
return jsonify({'error': form.time.errors }), 400
if not test:
flash('Invalid test ID.', 'error')
return redirect(url_for('admin._tests'))
return redirect(url_for('admin._tests', filter='active'))
return render_template('/admin/test.html', test = test, form = form)
@admin.route('/test/<string:id>/delete-adjustment/', methods=['POST'])
@@ -359,7 +357,7 @@ def _view_entries():
@admin.route('/results/<string:id>/', methods = ['GET', 'POST'])
@login_required
def _view_entry(id:str=None):
entry = Entry.query.filter_by(id=id)
entry = Entry.query.filter_by(id=id).first()
if request.method == 'POST':
if not entry: return jsonify({'error': 'Invalid entry ID.'}), 404
action = request.get_json()['action']
@@ -375,13 +373,14 @@ def _view_entry(id:str=None):
if not entry:
flash('Invalid entry ID.', 'error')
return redirect(url_for('admin._view_entries'))
test = entry['test']
test = entry.test
dataset = test.dataset
dataset_path = dataset.get_file()
with open(dataset_path, 'r') as _dataset:
data = loads(_dataset.read())
correct = get_correct_answers(dataset=data)
return render_template('/admin/result-detail.html', entry = entry, correct = correct)
answers = answer_options(dataset=data)
return render_template('/admin/result-detail.html', entry = entry, correct = correct, answers = answers)
@admin.route('/certificate/',methods=['POST'])
@login_required

4
ref-test/app/api/views.py
View File

@@ -23,7 +23,7 @@ def _fetch_questions():
if time_limit:
_time_limit = int(time_limit)
if user_code:
time_adjustment = test.time_adjustments[user_code]
time_adjustment = test.adjustments[user_code]
_time_limit += time_adjustment
end_delta = timedelta(minutes=_time_limit)
end_time = datetime.utcnow() + end_delta
@@ -40,7 +40,7 @@ def _fetch_questions():
return jsonify({
'time_limit': end_time,
'questions': questions,
'start_time': entry['start_time'],
'start_time': entry.start_time,
'time_adjustment': time_adjustment
}), 200

1
ref-test/app/models/dataset.py
View File

@@ -75,6 +75,7 @@ class Dataset(db.Model):
filename = secure_filename('.'.join([self.id,'json']))
file_path = path.join(data, 'questions', filename)
if not path.isfile(file_path): return False, 'Data file is missing.'
return True, 'Data file found.'
def get_file(self):
filename = secure_filename('.'.join([self.id,'json']))

13
ref-test/app/models/entry.py
View File

@@ -18,7 +18,7 @@ class Entry(db.Model):
club = db.Column(db.String(128), nullable=True)
test_id = db.Column(db.String(36), db.ForeignKey('test.id'))
user_code = db.Column(db.String(6), nullable=True)
start_time = db.Column(db.DateTime, nullable=False)
start_time = db.Column(db.DateTime, nullable=True)
end_time = db.Column(db.DateTime, nullable=True)
status = db.Column(db.String(16), nullable=True)
valid = db.Column(db.Boolean, default=True, nullable=True)
@@ -66,17 +66,24 @@ class Entry(db.Model):
def get_club(self): return decrypt(self.club)
def start(self):
def ready(self):
self.generate_id()
db.session.add(self)
db.session.commit()
write('tests.log', f'New test ready for {self.get_first_name()} {self.get_surname()}.')
return True, f'Test ready.'
def start(self):
self.start_time = datetime.now()
self.status = 'started'
write('tests.log', f'New test started by {self.get_first_name()} {self.get_surname()}.')
write('tests.log', f'Test started by {self.get_first_name()} {self.get_surname()}.')
db.session.commit()
return True, f'New test started with id {self.id}.'
def complete(self, answers:dict=None, result:dict=None):
self.end_time = datetime.now()
self.answers = answers
self.result = result
write('tests.log', f'Test completed by {self.get_first_name()} {self.get_surname()}.')
delta = timedelta(minutes=self.test.time_limit+1)
if not self.test.time_limit or self.end_time <= self.start_time + delta:

2
ref-test/app/quiz/templates/quiz/index.html
View File

@@ -13,7 +13,7 @@
The presentation of the questions is just a start, and we acknowledge we still have a long way to go. We welcome any feedback on how we can further improve this test.
</p>
<div class="button-container">
<a href="{{ url_for('quiz_views.instructions') }}" class="btn btn-success">
<a href="{{ url_for('quiz._instructions') }}" class="btn btn-success">
<i class="bi bi-book-fill button-icon"></i>
Read the Instructions
</a>

2
ref-test/app/quiz/templates/quiz/instructions.html
View File

@@ -53,7 +53,7 @@
</p>
</div>
<div class="button-container">
<a href="{{ url_for('quiz_views.start') }}" class="btn btn-success">
<a href="{{ url_for('quiz._start') }}" class="btn btn-success">
<i class="bi bi-pencil-fill button-icon"></i>
Take the Exam
</a>

4
ref-test/app/quiz/templates/quiz/result.html
View File

@@ -26,10 +26,10 @@
</div>
<div class="results-grade">
{{ entry.results.grade[0]|upper }}{{ entry.results.grade[1:] }}
{{ entry.result.grade[0]|upper }}{{ entry.result.grade[1:] }}
</div>
{% if entry.results.grade == 'fail' %}
{% if entry.result.grade == 'fail' %}
Unfortunately, you have not passed the theory exam in this attempt. For your next attempt, it might help to brush up on the following topics:
<ul>

34
ref-test/app/quiz/views.py
View File

@@ -11,7 +11,8 @@ quiz = Blueprint(
name='quiz',
import_name=__name__,
template_folder='templates',
static_folder='static'
static_folder='static',
static_url_path='/quiz/static'
)
@quiz.route('/')
@@ -24,7 +25,7 @@ def _home():
def _instructions():
return render_template('/quiz/instructions.html')
@quiz.route('/start/')
@quiz.route('/start/', methods=['GET', 'POST'])
def _start():
form = StartQuiz()
if request.method == 'POST':
@@ -33,8 +34,9 @@ def _start():
entry.set_first_name(request.form.get('first_name'))
entry.set_surname(request.form.get('surname'))
entry.set_club(request.form.get('club'))
entry.set_email(request.form.get('email'))
code = request.form.get('test_code').replace('—', '').lower()
test = Test.query.filter_by(code=code)
test = Test.query.filter_by(code=code).first()
entry.test = test
entry.user_code = request.form.get('user_code')
entry.user_code = None if entry.user_code == '' else entry.user_code.lower()
@@ -42,7 +44,7 @@ def _start():
if entry.user_code and entry.user_code not in test.adjustments: return jsonify({'error': f'The user code you entered is not valid.'}), 400
if test.end_date < datetime.now(): return jsonify({'error': f'The exam code you entered expired on {test["expiry_date"].strftime("%d %b %Y %H:%M")}.'}), 400
if test.start_date > datetime.now(): return jsonify({'error': f'The exam has not yet opened. Your exam code will be valid from {test["start_date"].strftime("%d %b %Y %H:%M")}.'}), 400
success, message = entry.start()
success, message = entry.ready()
if success:
session['id'] = entry.id
return jsonify({
@@ -52,36 +54,36 @@ def _start():
return jsonify({'error': 'There was an error processing the user test and/or user codes.'}), 400
errors = [*form.test_code.errors, *form.user_code.errors, *form.first_name.errors, *form.surname.errors, *form.email.errors, *form.club.errors]
return jsonify({ 'error': errors}), 400
render_template('/quiz/start_quiz.html', form = form)
return render_template('/quiz/start_quiz.html', form = form)
@quiz.route('/quiz/')
def _quiz():
id = session.get('id')
if not id or not Entry.query.filter_by(id=id).first():
flash('Your session was not recognised. Please sign in to the quiz again.', 'error')
session.pop('id')
return redirect(url_for('quiz_views.start'))
session.pop('id', None)
return redirect(url_for('quiz._start'))
return render_template('/quiz/client.html')
@quiz.route('/result/')
def _result():
id = session.get('id')
entry = Entry.query.filter_by('id').first()
if not entry:
return abort(404)
score = round(100*entry.results['score']/entry.results['max'])
tags_low = { tag: tag_result['max'] - tag_result['scored'] for tag, tag_result in entry.results['tags'].items() }
entry = Entry.query.filter_by(id=id).first()
if not entry: return abort(404)
session.pop('id',None)
score = round(100*entry.result['score']/entry.result['max'])
tags_low = { tag: tag_result['max'] - tag_result['scored'] for tag, tag_result in entry.result['tags'].items() }
sorted_low_tags = sorted(tags_low.items(), key=lambda x: x[1], reverse=True)
tag_output = [ tag[0] for tag in sorted_low_tags[0:3] if tag[1] > 3]
revision_plain = ''
revision_html = ''
if entry.results['grade'] == 'pass':
if entry.result['grade'] == 'pass':
flavour_text_plain = """Well done on successfully completing the refereeing theory exam. We really appreciate members of our community taking the time to get qualified to referee.
"""
elif entry.results['grade'] == 'merit':
elif entry.result['grade'] == 'merit':
flavour_text_plain = """Congratulations on achieving a merit in the refereeing exam. We are delighted that members of our community work so hard with refereeing.
"""
elif entry.results['grade'] == 'fail':
elif entry.result['grade'] == 'fail':
flavour_text_plain = """Unfortunately, you were not successful in passing the theory exam in this attempt. We hope that this does not dissuade you, and that you try again in the future.
"""
revision_plain = f"""Based on your answers, we would also suggest you brush up on the following topics for your next attempt:\n\n
@@ -92,6 +94,6 @@ def _result():
<li>{'</li><li>'.join(tag_output)}</li>
</ul>
"""
if not entry['status'] == 'late':
if not entry.status == 'late':
pass
return render_template('/quiz/result.html', entry=entry, score=score, tag_output=tag_output)

36
ref-test/app/tools/test.py
View File

@@ -20,7 +20,7 @@ def generate_questions(dataset:list):
'text': block['text']
}
if block['q_type'] == 'Multiple Choice': question['options'] = randomise_list([*enumerate(block['options'])])
else: question['options'] = block['options'].copy()
else: question['options'] = [*enumerate(block['options'])]
output.append(question)
elif block['type'] == 'block':
for key, _question in enumerate(randomise_list(block['questions'])):
@@ -33,7 +33,7 @@ def generate_questions(dataset:list):
'text': _question['text']
}
if _question['q_type'] == 'Multiple Choice': question['options'] = randomise_list([*enumerate(_question['options'])])
else: question['options'] = _question['options'].copy()
else: question['options'] = [*enumerate(_question['options'])]
output.append(question)
return output
@@ -66,6 +66,14 @@ def evaluate_answers(answers:dict, key:list):
'max': 1
}
else: tags[tag]['max'] += 1
else:
for tag in block['tags']:
if tag not in tags:
tags[tag] = {
'scored': 0,
'max': 1
}
else: tags[tag]['max'] += 1
elif block['type'] == 'block':
for question in block['questions']:
max += 1
@@ -91,6 +99,14 @@ def evaluate_answers(answers:dict, key:list):
'max': 1
}
else: tags[tag]['max'] += 1
else:
for tag in question['tags']:
if tag not in tags:
tags[tag] = {
'scored': 0,
'max': 1
}
else: tags[tag]['max'] += 1
grade = 'merit' if score/max >= .85 else 'pass' if score/max >= .70 else 'fail'
return {
'grade': grade,
@@ -103,10 +119,10 @@ def get_correct_answers(dataset:list):
output = {}
for block in dataset:
if block['type'] == 'question':
output[str(block['q_no'])] = block['options'][block['correct']]
output[str(block['q_no'])] = block['correct']
if block['type'] == 'block':
for question in block['questions']:
output[str(question['q_no'])] = question['options'][question['correct']]
output[str(question['q_no'])] = question['correct']
return output
def redirect_if_started(function):
@@ -117,3 +133,15 @@ def redirect_if_started(function):
return redirect(url_for('quiz._quiz'))
return function(*args, **kwargs)
return wrapper
def answer_options(dataset:list):
output = []
for block in dataset:
if block['type'] == 'question':
question = block['options'].copy()
output.append(question)
elif block['type'] == 'block':
for _question in block['questions']:
question = _question['options'].copy()
output.append(question)
return output