From 74f4cb49b0dbe8658aeddcd6d01cf089928dc4e9 Mon Sep 17 00:00:00 2001 From: Devin Matte Date: Mon, 3 Sep 2018 03:19:26 -0400 Subject: [PATCH 1/3] Aggressive Caching --- packet/ldap.py | 67 ++++++++++-------------------------------------- packet/packet.py | 4 +++ packet/utils.py | 37 +++----------------------- 3 files changed, 20 insertions(+), 88 deletions(-) diff --git a/packet/ldap.py b/packet/ldap.py index 65639621..564538e5 100644 --- a/packet/ldap.py +++ b/packet/ldap.py @@ -1,14 +1,14 @@ from functools import lru_cache -import ldap - from packet import _ldap +@lru_cache(maxsize=4096) def _ldap_get_group_members(group): return _ldap.get_group(group).get_members() +@lru_cache(maxsize=4096) def _ldap_is_member_of_group(member, group): group_list = member.get("memberOf") for group_dn in group_list: @@ -17,7 +17,7 @@ def _ldap_is_member_of_group(member, group): return False -@lru_cache(maxsize=1024) +@lru_cache(maxsize=2048) def _ldap_is_member_of_directorship(account, directorship): directors = _ldap.get_directorship_heads(directorship) for director in directors: @@ -48,16 +48,6 @@ def ldap_get_onfloor_members(): return _ldap_get_group_members("onfloor") -@lru_cache(maxsize=1024) -def ldap_get_current_students(): - return _ldap_get_group_members("current_student") - - -@lru_cache(maxsize=1024) -def ldap_get_all_members(): - return _ldap_get_group_members("member") - - @lru_cache(maxsize=1024) def ldap_get_groups(account): group_list = account.get("memberOf") @@ -68,17 +58,6 @@ def ldap_get_groups(account): return groups -@lru_cache(maxsize=1024) -def ldap_get_group_desc(group): - con = _ldap.get_con() - results = con.search_s( - "cn=groups,cn=accounts,dc=csh,dc=rit,dc=edu", - ldap.SCOPE_SUBTREE, - "(cn=%s)" % group, - ['description']) - return results[0][1]['description'][0].decode('utf-8') - - @lru_cache(maxsize=1024) def ldap_get_eboard(): members = _ldap_get_group_members("eboard-chairman") + _ldap_get_group_members("eboard-evaluations" @@ -90,6 +69,7 @@ def ldap_get_eboard(): return members +@lru_cache(maxsize=2048) def ldap_get_live_onfloor(): """ :return: All upperclassmen who live on floor and are not eboard @@ -101,67 +81,46 @@ def ldap_get_live_onfloor(): members.append(member) return members + # Status checkers +@lru_cache(maxsize=1024) def ldap_is_active(account): return _ldap_is_member_of_group(account, 'active') +@lru_cache(maxsize=1024) def ldap_is_alumni(account): # If the user is not active, they are an alumni. return not _ldap_is_member_of_group(account, 'active') +@lru_cache(maxsize=1024) def ldap_is_eboard(account): return _ldap_is_member_of_group(account, 'eboard') +@lru_cache(maxsize=1024) def ldap_is_rtp(account): return _ldap_is_member_of_group(account, 'rtp') +@lru_cache(maxsize=1024) def ldap_is_intromember(account): return _ldap_is_member_of_group(account, 'intromembers') +@lru_cache(maxsize=1024) def ldap_is_onfloor(account): return _ldap_is_member_of_group(account, 'onfloor') +@lru_cache(maxsize=1024) def ldap_is_current_student(account): return _ldap_is_member_of_group(account, 'current_student') -# Directorships - -def ldap_is_financial_director(account): - return _ldap_is_member_of_directorship(account, 'financial') - - -def ldap_is_eval_director(account): - return _ldap_is_member_of_directorship(account, 'evaluations') - - -def ldap_is_chairman(account): - return _ldap_is_member_of_directorship(account, 'chairman') - - -def ldap_is_history(account): - return _ldap_is_member_of_directorship(account, 'history') - - -def ldap_is_imps(account): - return _ldap_is_member_of_directorship(account, 'imps') - - -def ldap_is_social(account): - return _ldap_is_member_of_directorship(account, 'Social') - - -def ldap_is_rd(account): - return _ldap_is_member_of_directorship(account, 'research') - - +@lru_cache(maxsize=1024) def ldap_get_roomnumber(account): try: return account.roomNumber diff --git a/packet/packet.py b/packet/packet.py index 38490fdb..411c6170 100644 --- a/packet/packet.py +++ b/packet/packet.py @@ -1,3 +1,5 @@ +from functools import lru_cache + from packet.ldap import ldap_get_member, ldap_is_intromember from .models import Freshman, UpperSignature, FreshSignature, MiscSignature, db @@ -48,9 +50,11 @@ def get_signatures(freshman_username): 'misc': misc_signatures} +@lru_cache(maxsize=512) def get_number_signed(freshman_username): return Freshman.query.filter_by(rit_username=freshman_username).first().current_packet().signatures_received() +@lru_cache(maxsize=4096) def get_number_required(freshman_username): return Freshman.query.filter_by(rit_username=freshman_username).first().current_packet().signatures_required() diff --git a/packet/utils.py b/packet/utils.py index 5eceb93c..6eb58ac1 100644 --- a/packet/utils.py +++ b/packet/utils.py @@ -45,6 +45,7 @@ def wrapped_function(*args, **kwargs): return wrapped_function +@lru_cache(maxsize=2048) def get_member_info(uid): account = ldap_get_member(uid) @@ -52,48 +53,14 @@ def get_member_info(uid): "user_obj": account, "group_list": ldap_get_groups(account), "uid": account.uid, - "ritUid": parse_rit_uid(account.ritDn), "name": account.cn, "active": ldap_is_active(account), "onfloor": ldap_is_onfloor(account), "room": ldap_get_roomnumber(account), - "hp": account.housingPoints, - "plex": account.plex, - "rn": ldap_get_roomnumber(account), - "birthday": parse_date(account.birthday), - "memberSince": parse_date(account.memberSince), - "lastlogin": parse_date(account.krblastsuccessfulauth), - "year": parse_account_year(account.memberSince) } return member_info -def parse_date(date): - if date: - year = date[0:4] - month = date[4:6] - day = date[6:8] - return month + "-" + day + "-" + year - return False - - -def parse_rit_uid(dn): - if dn: - return dn.split(",")[0][4:] - - return None - - -def parse_account_year(date): - if date: - year = int(date[0:4]) - month = int(date[4:6]) - if month <= 8: - year = year - 1 - return year - return None - - @lru_cache(maxsize=2048) def is_on_floor(uid): return (Freshman.query.filter_by(rit_username=uid)).first().onfloor @@ -118,6 +85,7 @@ def signed_packet(signer, freshman): @app.context_processor def utility_processor(): # pylint: disable=bare-except + @lru_cache(maxsize=4096) def get_display_name(username): try: member = ldap_get_member(username) @@ -125,6 +93,7 @@ def get_display_name(username): except: return username + @lru_cache(maxsize=4096) def get_freshman_name(username): try: freshman = Freshman.query.filter_by(rit_username=username).first() From d55474720f1eb401db9b3bb5589c667c54faa9d9 Mon Sep 17 00:00:00 2001 From: Devin Matte Date: Mon, 3 Sep 2018 15:21:47 -0400 Subject: [PATCH 2/3] Removing unnecessary functions and clearing cache --- packet/ldap.py | 11 ++++++++--- packet/packet.py | 7 ++++++- packet/routes/shared.py | 3 ++- packet/templates/active_packets.html | 2 +- packet/templates/packet.html | 2 +- packet/templates/upperclassman.html | 2 +- packet/utils.py | 10 +--------- 7 files changed, 20 insertions(+), 17 deletions(-) diff --git a/packet/ldap.py b/packet/ldap.py index 564538e5..1e4f153a 100644 --- a/packet/ldap.py +++ b/packet/ldap.py @@ -3,12 +3,12 @@ from packet import _ldap -@lru_cache(maxsize=4096) +@lru_cache(maxsize=1024) def _ldap_get_group_members(group): return _ldap.get_group(group).get_members() -@lru_cache(maxsize=4096) +@lru_cache(maxsize=2048) def _ldap_is_member_of_group(member, group): group_list = member.get("memberOf") for group_dn in group_list: @@ -17,7 +17,7 @@ def _ldap_is_member_of_group(member, group): return False -@lru_cache(maxsize=2048) +@lru_cache(maxsize=1024) def _ldap_is_member_of_directorship(account, directorship): directors = _ldap.get_directorship_heads(directorship) for director in directors: @@ -84,6 +84,11 @@ def ldap_get_live_onfloor(): # Status checkers +@lru_cache(maxsize=1024) +def ldap_is_eval_director(account): + return _ldap_is_member_of_directorship(account, 'evaluations') + + @lru_cache(maxsize=1024) def ldap_is_active(account): return _ldap_is_member_of_group(account, 'active') diff --git a/packet/packet.py b/packet/packet.py index 411c6170..727892cb 100644 --- a/packet/packet.py +++ b/packet/packet.py @@ -34,9 +34,14 @@ def sign(signer_username, freshman_username): db.session.add(MiscSignature(packet=packet, member=signer_username)) db.session.commit() + # Clear functions that read signatures cache + get_number_signed.cache_clear() + get_signatures.cache_clear() + return True +@lru_cache(maxsize=2048) def get_signatures(freshman_username): packet = Freshman.query.filter_by(rit_username=freshman_username).first().current_packet() eboard = UpperSignature.query.filter_by(packet_id=packet.id, eboard=True).order_by(UpperSignature.signed.desc()) @@ -50,7 +55,7 @@ def get_signatures(freshman_username): 'misc': misc_signatures} -@lru_cache(maxsize=512) +@lru_cache(maxsize=2048) def get_number_signed(freshman_username): return Freshman.query.filter_by(rit_username=freshman_username).first().current_packet().signatures_received() diff --git a/packet/routes/shared.py b/packet/routes/shared.py index ef4c8061..4c687697 100644 --- a/packet/routes/shared.py +++ b/packet/routes/shared.py @@ -1,7 +1,8 @@ -from flask import render_template from datetime import datetime from itertools import chain +from flask import render_template + from packet import auth, app from packet.models import Freshman, Packet from packet.packet import get_signatures, get_number_required, get_number_signed diff --git a/packet/templates/active_packets.html b/packet/templates/active_packets.html index 3a6bf969..103cd11d 100644 --- a/packet/templates/active_packets.html +++ b/packet/templates/active_packets.html @@ -33,7 +33,7 @@

Active Packets

alt="{{ packet.freshman.name }}" src="https://www.gravatar.com/avatar/freshmen?d=mp&f=y" width="25" - height="25"/> {{ get_freshman_name(packet.freshman.rit_username) }} + height="25"/> {{ packet.freshman.name }} ({{ packet.freshman.rit_username }}) diff --git a/packet/templates/packet.html b/packet/templates/packet.html index 1e3ed056..0a3b7518 100644 --- a/packet/templates/packet.html +++ b/packet/templates/packet.html @@ -166,7 +166,7 @@
Upperclassmen Score
{{ m.freshman.rit_username }} {{ get_freshman_name(m.freshman.rit_username) }} + height="25"/> {{ m.freshman.name }} ({{ m.freshman.rit_username }}) diff --git a/packet/templates/upperclassman.html b/packet/templates/upperclassman.html index 22fdc60d..d352cc86 100644 --- a/packet/templates/upperclassman.html +++ b/packet/templates/upperclassman.html @@ -32,7 +32,7 @@

{{ get_display_name(member) }}

alt="{{ m.packet.freshman.name }}" src="https://profiles.csh.rit.edu/image/{{ m.packet.freshman.rit_username }}" width="25" - height="25"/> {{ get_freshman_name(m.packet.freshman.name) }} + height="25"/> {{ m.packet.freshman.name }} ({{ m.packet.freshman.rit_username }}) diff --git a/packet/utils.py b/packet/utils.py index 6eb58ac1..67730aec 100644 --- a/packet/utils.py +++ b/packet/utils.py @@ -93,12 +93,4 @@ def get_display_name(username): except: return username - @lru_cache(maxsize=4096) - def get_freshman_name(username): - try: - freshman = Freshman.query.filter_by(rit_username=username).first() - return freshman.name + " (" + freshman.rit_username + ")" - except: - return username - - return dict(get_display_name=get_display_name, get_freshman_name=get_freshman_name) + return dict(get_display_name=get_display_name) From 2ddcb1d5e996cb7919f95398aa042f5f44fa5c64 Mon Sep 17 00:00:00 2001 From: Devin Matte Date: Mon, 3 Sep 2018 16:04:37 -0400 Subject: [PATCH 3/3] Caching and cleaning functions --- packet/models.py | 3 +++ packet/packet.py | 15 +++++++++++++++ packet/routes/shared.py | 25 ++++++++----------------- 3 files changed, 26 insertions(+), 17 deletions(-) diff --git a/packet/models.py b/packet/models.py index 5bd3f0a5..62f07de2 100644 --- a/packet/models.py +++ b/packet/models.py @@ -3,6 +3,8 @@ """ from datetime import datetime +from functools import lru_cache + from sqlalchemy import Column, Integer, String, Text, ForeignKey, DateTime, Boolean from sqlalchemy.orm import relationship @@ -47,6 +49,7 @@ class Packet(db.Model): def is_open(self): return self.start < datetime.now() < self.end + @lru_cache(maxsize=1024) def signatures_required(self): eboard = UpperSignature.query.filter_by(eboard=True).count() return {'eboard': eboard, diff --git a/packet/packet.py b/packet/packet.py index 727892cb..a0827237 100644 --- a/packet/packet.py +++ b/packet/packet.py @@ -1,3 +1,4 @@ +import copy from functools import lru_cache from packet.ldap import ldap_get_member, ldap_is_intromember @@ -37,6 +38,7 @@ def sign(signer_username, freshman_username): # Clear functions that read signatures cache get_number_signed.cache_clear() get_signatures.cache_clear() + get_upperclassmen_percent.cache_clear() return True @@ -63,3 +65,16 @@ def get_number_signed(freshman_username): @lru_cache(maxsize=4096) def get_number_required(freshman_username): return Freshman.query.filter_by(rit_username=freshman_username).first().current_packet().signatures_required() + + +@lru_cache(maxsize=2048) +def get_upperclassmen_percent(uid): + upperclassmen_required = copy.deepcopy(get_number_required(uid)) + del upperclassmen_required['freshmen'] + upperclassmen_required = sum(upperclassmen_required.values()) + + upperclassmen_signature = copy.deepcopy(get_number_signed(uid)) + del upperclassmen_signature['freshmen'] + upperclassmen_signature = sum(upperclassmen_signature.values()) + + return upperclassmen_signature / upperclassmen_required * 100 diff --git a/packet/routes/shared.py b/packet/routes/shared.py index 4c687697..e89401ea 100644 --- a/packet/routes/shared.py +++ b/packet/routes/shared.py @@ -5,7 +5,7 @@ from packet import auth, app from packet.models import Freshman, Packet -from packet.packet import get_signatures, get_number_required, get_number_signed +from packet.packet import get_signatures, get_number_required, get_number_signed, get_upperclassmen_percent from packet.utils import before_request, signed_packet @@ -14,21 +14,12 @@ @before_request def freshman_packet(uid, info=None): freshman = Freshman.query.filter_by(rit_username=uid).first() + upperclassmen_percent = get_upperclassmen_percent(uid) signatures = get_signatures(uid) signed_dict = get_number_signed(uid) required = sum(get_number_required(uid).values()) signed = sum(signed_dict.values()) - upperclassmen_required = get_number_required(uid) - del upperclassmen_required['freshmen'] - upperclassmen_required = sum(upperclassmen_required.values()) - - upperclassmen_signature = get_number_signed(uid) - del upperclassmen_signature['freshmen'] - upperclassmen_signature = sum(upperclassmen_signature.values()) - - upperclassmen_percent = upperclassmen_signature / upperclassmen_required * 100 - packet_signed = signed_packet(info['uid'], uid) return render_template("packet.html", info=info, signatures=signatures, uid=uid, required=required, signed=signed, freshman=freshman, packet_signed=packet_signed, upperclassmen_percent=upperclassmen_percent, @@ -39,12 +30,12 @@ def freshman_packet(uid, info=None): @auth.oidc_auth @before_request def packets(info=None): - packets = Packet.query.filter(Packet.end > datetime.now()).filter(Packet.start < datetime.now()).all() + open_packets = Packet.query.filter(Packet.end > datetime.now()).filter(Packet.start < datetime.now()).all() # Add the did_sign flag if app.config["REALM"] == "csh": # User is an upperclassman - for packet in packets: + for packet in open_packets: packet.did_sign = False packet.total_signatures = sum(packet.signatures_received().values()) packet.required_signatures = sum(packet.signatures_required().values()) @@ -55,7 +46,7 @@ def packets(info=None): break else: # User is a freshman - for packet in packets: + for packet in open_packets: packet.did_sign = False packet.total_signatures = sum(packet.signatures_received().values()) packet.required_signatures = sum(packet.signatures_required().values()) @@ -65,7 +56,7 @@ def packets(info=None): packet.did_sign = True break - packets.sort(key=lambda x: sum(x.signatures_received().values()), reverse=True) - packets.sort(key=lambda x: x.did_sign, reverse=True) + open_packets.sort(key=lambda x: sum(x.signatures_received().values()), reverse=True) + open_packets.sort(key=lambda x: x.did_sign, reverse=True) - return render_template("active_packets.html", info=info, packets=packets) + return render_template("active_packets.html", info=info, packets=open_packets)