diff --git a/sw/python/.gitignore b/sw/python/.gitignore new file mode 100644 index 0000000..b6e4761 --- /dev/null +++ b/sw/python/.gitignore @@ -0,0 +1,129 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +pip-wheel-metadata/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/sw/python/aionfpga_daa.py b/sw/python/aionfpga_daa.py new file mode 100644 index 0000000..358115e --- /dev/null +++ b/sw/python/aionfpga_daa.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ data acquisition application (daa) +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import os +import shutil +import subprocess +from time import time + +import modules.daa as daa + +# location of the camera interface binary +cam_interface = r'..\cam-interface\cam-interface.exe' + +def main(): + try: + throwid = daa.fetch_field(daa.tab_settings, 0, 'throwid') + except Exception as e: + print(e) + input() + raise e + + session_throwid = 1 + + for f in os.listdir(daa.dir_temp): + os.unlink(daa.dir_temp / f) + + while True: + daa.clear() + + print(f'Throw #{session_throwid} ({throwid} in total)\n') + print(r''' + _____ _ + | __ \ | | + | |__) |___ __ _ __| |_ _ + | _ // _ \/ _` |/ _` | | | | + | | \ \ __/ (_| | (_| | |_| | + |_| \_\___|\__,_|\__,_|\__, | + __/ | + |___/ + ''') + + subp = [cam_interface] + res = subprocess.run(subp, stdout=subprocess.PIPE) + # stdout = res.stdout.decode('utf-8') + + daa.clear() + + print(f'Throw #{session_throwid} ({throwid} in total)\n') + + valid = daa.choice('Throw valid?', 'No', 'Yes') + print('\n') + + if valid: + inp = daa.print_objects_list() + print('\n') + + throwid += 1 + session_throwid += 1 + + ts = int(time()) + obj = daa.objects[inp] + obj_san = daa.objects_san[inp] + framegood = True + partial = False + + img_name = lambda i : f'{i}.png' + frame = lambda i : f'{ts}_{throwid}_{i}_{obj_san}.png' + + rows = [] + for idx in range(1, len(os.listdir(daa.dir_temp))+1): + row = [ts, throwid, idx, f"'{frame(idx)}'", f"'{obj}'", framegood, partial] + rows.append(row) + + daa.insert_rows(daa.tab_frames, rows) + + for idx, f in enumerate(os.listdir(daa.dir_temp)): + shutil.move(daa.dir_temp / img_name(idx), + daa.dir_frames / frame(idx+1)) + + try: + daa.update_field(daa.tab_settings, 0, 'throwid', throwid) + except Exception as e: + print(e) + input() + raise e + else: + for f in os.listdir(daa.dir_temp): + os.unlink(daa.dir_temp / f) + +if __name__ == '__main__': + main() diff --git a/sw/python/aionfpga_db_backup.py b/sw/python/aionfpga_db_backup.py new file mode 100644 index 0000000..e3f8e98 --- /dev/null +++ b/sw/python/aionfpga_db_backup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ db backup +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import csv +from time import time + +import modules.daa as daa + +def backup_db(): + columns = daa.fetch_rows_and('COLUMNS', ['TABLE_SCHEMA', 'TABLE_NAME'], + [daa.database, daa.tab_frames], 'COLUMN_NAME', + database='INFORMATION_SCHEMA') + columns = [column[0] for column in columns] + rows = daa.fetch_all_rows(daa.tab_frames) + timestamp = int(time()) + with open(f'aionfpga_db_{timestamp}.csv', 'w', newline='') as csv_file: + csv_writer = csv.writer(csv_file, delimiter=',') + csv_writer.writerow(columns) + csv_writer.writerows(rows) + return timestamp + +def print_db(timestamp): + with open(f'aionfpga_db_{timestamp}.csv', 'r', newline='') as csv_file: + csv_reader = csv.reader(csv_file, delimiter=',') + for row in csv_reader: + print(row) + +def main(): + ts = backup_db() + print_db(ts) + +if __name__ == '__main__': + main() diff --git a/sw/python/aionfpga_fix_frameid.py b/sw/python/aionfpga_fix_frameid.py new file mode 100644 index 0000000..04fec3a --- /dev/null +++ b/sw/python/aionfpga_fix_frameid.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ fix frameid +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import os +import re +import shutil + +import modules.daa as daa + +directory = daa.dir_frames + +# Functions for unit-testing + +test_ext = '.txt' + +test_ts = 1574937102 +test_tid = 1 +test_object = 'spiky-ball' +test_name = lambda i : f'{test_ts}_{test_tid}_{i}_{test_object}' + +def unit_test(): + for i in range(100): + create_unordered_files(i+1) + fix_frameid(test_ts, test_tid, i+1, test_object, test_ext) + res = check_files() + if res: + if res is not True: + print(f"Fixing {i+1} didn't work.") + break + else: + pass # no error + else: + print(f'Error in {i+1} (RegEx).') + break + +def create_unordered_files(amount): + # remove everything in the frames directory + for f in os.listdir(directory): + os.unlink(f'{directory}\\{f}') + + for i in range(amount): + with open(f'{directory}\\{i}', 'w') as f: + f.write(str(i)) + + for (i, f) in enumerate(os.listdir(directory)): + shutil.move(f'{directory}\\{f}', f'{directory}\\{test_name(i+1)}{test_ext}') + +def check_files(): + for file in os.listdir(directory): + with open(f'{directory}\\{file}', 'r') as f: + frameid_search = re.search("^[\\d]*_[\\d]*_([\\d]*)_[a-z-]*.[a-z]*$", file) + if frameid_search: + frameid = int(frameid_search.group(1)) + else: + return None + text = f.read() + if int(frameid) != int(text)+1: + return False + return True + +# Functions to fix the frameid + +def move_file(name_1, name_2): + shutil.move(f'{directory}\\{name_1}', f'{directory}\\{name_2}') + +# fix up to 100 frames +def fix_frameid(timestamp, throwid, frames, object_safe, file_ext='.png'): + nof = frames + rest_list_shift = [0 for x in range(nof) if x < 2] + rest_list = [x+1 for x in range(nof) if x < 2] + + tmp_ext = '.tmp' + name = lambda i : f'{timestamp}_{throwid}_{i}_{object_safe}' + + # add '.tmp' to all files + for i in range(frames): + move_file(f'{name(i+1)}{file_ext}', f'{name(i+1)}{file_ext}{tmp_ext}') + + if nof > 10: + packets = [(p*11 + 3) for p in range(int((nof-10)/10))] # start of full packet + rest_list_shift.extend([10*(i+1) for (i, p) in enumerate(packets) if p+10 < 100]) + rest_list.extend([p+10 for p in packets if p+10 < 100]) + + shift = 8 + for i in packets: # handle full packets + for j in range(10): + move_file(f'{name(i+j)}{file_ext}{tmp_ext}', f'{name(i+j+shift)}{file_ext}') + shift = shift - 1 + + nof_left = nof - len(packets)*10 - 10 # len(packets)*10: handled by full packages / 10: 1-10 + if nof_left > 0: + index = len(packets)*11 + 3 + rest_list_shift.extend([rest_list_shift[len(rest_list_shift)-1]+nof_left]*(10-len(rest_list_shift))) + rest_list.extend(range(index+nof_left, nof+1)) + for i in range(nof_left): # handle not full packets + move_file(f'{name(index)}{file_ext}{tmp_ext}', f'{name(index+shift)}{file_ext}') + index = index + 1 + else: + rest_list_shift.extend([rest_list_shift[len(rest_list_shift)-1]]*(10-len(rest_list_shift))) + rest_list.extend(range(len(packets)*10+len(rest_list)+1, nof+1)) + else: + rest_list_shift.extend([0 for x in range(nof-len(rest_list))]) + rest_list.extend([x+3 for x in range(nof-len(rest_list))]) + + for (i, r) in enumerate(rest_list): # handle the lowest + move_file(f'{name(r)}{file_ext}{tmp_ext}', f'{name(r-rest_list_shift[i])}{file_ext}') + +def main(): + num_of_frames = 0 + try: + throws = daa.fetch_field(daa.tab_settings, 0, 'throwid') + except Exception as e: + print(e) + input() + raise e + for i in range(throws): + try: + throw = daa.fetch_rows(daa.tab_frames, 'throwid', i+1, '*') + except Exception as e: + print(e) + input() + raise e + rowcount = len(throw) + ts = throw[0][1] + obj = throw[0][5] + obj_san = daa.objects_san[daa.objects.index(obj)] + fix_frameid(ts, i+1, rowcount, obj_san) + num_of_frames += rowcount + print(f'{i+1}: {ts} {obj_san} {rowcount}') + print(f'#Frames = {num_of_frames}') + +if __name__ == '__main__': + # unit_test() + main() diff --git a/sw/python/aionfpga_partial_frames.py b/sw/python/aionfpga_partial_frames.py new file mode 100644 index 0000000..c015e59 --- /dev/null +++ b/sw/python/aionfpga_partial_frames.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ partial frames +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import os +import shutil + +import modules.daa as daa + +# daa.Users.DOMINIK or daa.Users.NICO +user = daa.User.DOMINIK + +def main(): + try: + throwid = daa.fetch_field(daa.tab_settings, 0, user.name.lower()) + except Exception as e: + print(e) + input() + raise e + + while True: + daa.clear() + + for f in os.listdir(daa.dir_temp): + os.unlink(daa.dir_temp / f) + + print(f'Throw #{throwid}\n') + + try: + throw = daa.fetch_rows(daa.tab_frames, 'throwid', throwid, '*') + except Exception as e: + print(e) + input() + raise e + + id = throw[0][0] + rowcount = len(throw) + + ts = throw[0][1] + frame = throw[0][4] + obj = throw[0][5] + obj_san = daa.objects_san[daa.objects.index(obj)] + + name = lambda i : f'{ts}_{throwid}_{i}_{obj_san}.png' + + for idx in range(1, rowcount+1): + shutil.copy(daa.dir_frames / name(idx), + daa.dir_temp / name(idx)) + + inp = daa.input_integer_list("Enter frameid's (partial frames):", rowcount) + inp = sorted(inp) + + input(f'\nHit enter to commit `{inp}` ') + + ids = inp.copy() + values = [1]*len(ids) + for idx in range(rowcount): + if idx+1 not in inp: + ids.append(idx+1) + values.append(0) + ids[idx] += id - 1 + + try: + daa.update_fields(daa.tab_frames, ids, 'partial', values) + except Exception as e: + print(e) + input() + raise e + + if user == daa.User.DOMINIK: + throwid -= 1 + else: + throwid += 1 + + try: + daa.update_field(daa.tab_settings, 0, user.name.lower(), throwid) + except Exception as e: + print(e) + input() + raise e + +if __name__ == '__main__': + main() diff --git a/sw/python/aionfpga_review_labels.py b/sw/python/aionfpga_review_labels.py new file mode 100644 index 0000000..843d070 --- /dev/null +++ b/sw/python/aionfpga_review_labels.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ review labels +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import os +import shutil + +import modules.daa as daa + +# daa.Users.DOMINIK or daa.Users.NICO +user = daa.User.DOMINIK + +def main(): + try: + throwid = daa.fetch_field(daa.tab_settings, 0, user.name.lower()) + except Exception as e: + print(e) + input() + raise e + + while True: + daa.clear() + + for f in os.listdir(daa.dir_temp): + os.unlink(daa.dir_temp / f) + + print(f'Throw #{throwid}\n') + + try: + throw = daa.fetch_rows(daa.tab_frames, 'throwid', throwid, '*') + except Exception as e: + print(e) + input() + raise e + + id = throw[0][0] + rowcount = len(throw) + + ts = throw[0][1] + frame = throw[0][4] + obj_old = throw[0][5] + obj_san_old = daa.objects_san[daa.objects.index(obj_old)] + + frame_old = lambda i : f'{ts}_{throwid}_{i}_{obj_san_old}.png' + + for idx in range(1, rowcount+1): + shutil.copy(daa.dir_frames / frame_old(idx), + daa.dir_temp / frame_old(idx)) + + print(f'Current throw is labeled as: `{obj_old}`\n\n') + + correct = daa.choice('Correct?', 'No', 'Yes') + print('\n') + + if not correct: + inp = daa.print_objects_list() + print('\n') + + obj_new = daa.objects[inp] + obj_new_san = daa.objects_san[inp] + + changed = (obj_new != obj_old) + + if not changed: + input(f'Hit enter to abort without changing the label (`{obj_old}` stays `{obj_new}`) ') + else: + name_new = lambda i : f'{ts}_{throwid}_{i}_{obj_new_san}.png' + + # option to abort before committing + input(f'Hit enter to commit `{obj_new}` ') + + ids = [id + idx for idx in range(rowcount)] + frames = [name_new(idx) for idx in range(1, rowcount+1)] + + try: + daa.update_fields(daa.tab_frames, ids, 'frame', frames) + daa.update_fields(daa.tab_frames, ids, 'object', [obj_new]*rowcount) + except Exception as e: + print(e) + input() + raise e + + for idx in range(1, rowcount+1): + shutil.move(daa.dir_frames / frame_old(idx), + daa.dir_frames / name_new(idx)) + + if user == daa.User.DOMINIK: + throwid -= 1 + else: + throwid += 1 + + try: + daa.update_field(daa.tab_settings, 0, user.name.lower(), throwid) + except Exception as e: + print(e) + input() + raise e + +if __name__ == '__main__': + main() diff --git a/sw/python/aionfpga_stats.py b/sw/python/aionfpga_stats.py new file mode 100644 index 0000000..b5d568d --- /dev/null +++ b/sw/python/aionfpga_stats.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python3 + +'''aionfpga ~ stats +Copyright (C) 2019 Dominik Müller and Nico Canzani +''' + +import matplotlib.pyplot as plt + +import modules.daa as daa + +objects_plot = [ + 'Dart', 'Football', 'Ping Pong', 'Shuttle', 'Sporf', 'Arrow', + 'Featherball', 'Floorball', 'Spiky', 'Tesafilm', 'Sponge', 'Duplo (Red)', + 'Duplo (Green)', 'Duplo Figure', 'Dice', 'Shoe', 'Bunny', 'Glove', 'Cord', + 'Paper', 'Beer Cap', 'Bottle' +] + +def main(): + noo, *_ = daa.objects_list_info() + + frames = [0]*noo + frames_partial = [0]*noo + + rows = daa.fetch_all_rows(daa.tab_frames) + for row in rows: + if row[6] == 1: # valid + if row[7] == 0: # not partial + frames[daa.objects.index(row[5])] += 1 + else: + frames_partial[daa.objects.index(row[5])] += 1 + + frames_valid = sum(frames) + sum(frames_partial) + frames_invalid = len(rows) - frames_valid + + print(f'frames_valid {frames_valid}') + print(f'frames_invalid {frames_invalid}') + + for idx, o in enumerate(daa.objects): + if frames[idx] < 500: + print(f'{o}: {frames[idx]}') + + plt.figure(figsize=[12, 6], tight_layout=True) + p1 = plt.bar(objects_plot, frames, color=(0, 0.4470, 0.7410)) + p2 = plt.bar(objects_plot, frames_partial, bottom=frames, color=(0.8500, 0.3250, 0.0980)) + plt.legend((p1[0], p2[0]), ('Object fully visible', 'Object partially visible')) + plt.xticks(rotation=45) + plt.savefig('statistics.pdf', transparent=True) + plt.show() + +if __name__ == '__main__': + main() diff --git a/sw/python/modules/definitions.py b/sw/python/modules/definitions.py index 1b334b8..cedf616 100644 --- a/sw/python/modules/definitions.py +++ b/sw/python/modules/definitions.py @@ -5,6 +5,7 @@ ''' import os +import re import math import MySQLdb @@ -32,9 +33,51 @@ def input_integer_range(text, min, max): if inp >= min and inp <= max: return inp -# input space separated list of numbers -def input_integer_list(): - pass +# input space separated list of integers +def input_integer_list(text, max): + inp = None + while inp is None: + print() + inp = input(f'{text} ').strip().split() + if inp == []: + break + elif inp[0] == 'all': + inp = [i+1 for i in range(max)] + break + inp_tmp = inp.copy() + inp_ext = [] + popped = 0 + for idx, i in enumerate(inp): + search = re.search("^([\\d]+)-([\\d]+)$", i) + if search: + inp_tmp.pop(idx-popped) + inp_ext.extend(list(range(int(search.group(1)), + int(search.group(2))+1))) + popped += 1 + inp_tmp.extend(inp_ext) + inp = inp_tmp + try: + inp = map(int, inp) + inp = list(inp) + except Exception as e: + print(f'Error: only numbers allowed') + inp = None + else: + inp = list(set(inp)) + + if len(inp) > max: + print(f'Error: input > frames ({len(inp)} > {max})') + inp = None + else: + for i in inp: + if i < 1 or i > max: + if i < 1: + print(f'Error: input < 1 ({i} < 1)') + else: + print(f'Error: input > max(input) ({i} > {max})') + inp = None + break + return inp # return infos about the `objects` list def objects_list_info():