Skip to content

Commit

Permalink
lint + clean up prints
Browse files Browse the repository at this point in the history
  • Loading branch information
AaDalal committed Feb 6, 2024
1 parent 324d2bf commit d390f17
Show file tree
Hide file tree
Showing 8 changed files with 112 additions and 72 deletions.
47 changes: 25 additions & 22 deletions backend/degree/management/commands/deduplicate_rules.py
Original file line number Diff line number Diff line change
@@ -1,24 +1,27 @@
import decimal
import json
from collections import OrderedDict, defaultdict
from textwrap import dedent

from django.core.management.base import BaseCommand, CommandParser
from django.core.management.base import BaseCommand
from django.db import transaction
from degree.models import Rule, Degree
from degree.serializers import RuleSerializer
import json
from collections import defaultdict, OrderedDict
import decimal
from tqdm import tqdm
from pprint import pprint

from degree.models import Degree, Rule
from degree.serializers import RuleSerializer


class DecimalEncoder(json.JSONEncoder):
"""
JSON encoder that can handle Decimal objects
"""

def default(self, o):
if isinstance(o, decimal.Decimal):
return str(o)
return super().default(o)


def recursively_pop(d: dict(), keys: list[str]) -> None:
"""
Recursively remove keys from a dictionary
Expand All @@ -30,22 +33,25 @@ def recursively_pop(d: dict(), keys: list[str]) -> None:
if isinstance(value, dict) or isinstance(value, OrderedDict):
recursively_pop(value, keys)

def deduplicate_rules(verbose=False):

def deduplicate_rules(verbose=False):
rule_to_hash = dict()
for rule in tqdm(Rule.objects.all(), disable=not verbose, desc="Hashing rules"):
serialized = RuleSerializer(rule).data # recursively serializes the rule
serialized = RuleSerializer(rule).data # recursively serializes the rule
recursively_pop(serialized, keys=["id", "parent"])
rule_to_hash[rule.id] = hash(json.dumps(
serialized,
sort_keys=True,
ensure_ascii=True,
cls=DecimalEncoder,
))
rule_to_hash[rule.id] = hash(
json.dumps(
serialized,
sort_keys=True,
ensure_ascii=True,
cls=DecimalEncoder,
)
)

hash_to_rule = defaultdict(list)
for rule_id, hashed in rule_to_hash.items():
hash_to_rule[hashed].append(rule_id)

delete_count = 0
for rule_ids in tqdm(hash_to_rule.values(), disable=not verbose, desc="Deleting duplicates"):
if len(rule_ids) > 1:
Expand All @@ -55,9 +61,10 @@ def deduplicate_rules(verbose=False):
degree.rules.remove(*rule_ids[1:])
deleted, _ = Rule.objects.filter(id__in=rule_ids[1:]).delete()
delete_count += deleted

return delete_count



class Command(BaseCommand):
help = dedent(
"""
Expand All @@ -70,7 +77,3 @@ def handle(self, *args, **kwargs):
delete_count = deduplicate_rules(verbose=kwargs["verbosity"])
if kwargs["verbosity"]:
print(f"Deleted {delete_count} duplicate rules")




14 changes: 8 additions & 6 deletions backend/degree/management/commands/fetch_degrees.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@

from django.core.management.base import BaseCommand
from django.db import transaction
from degree.management.commands.deduplicate_rules import deduplicate_rules

from courses.util import get_current_semester
from degree.management.commands.deduplicate_rules import deduplicate_rules
from degree.models import Degree, program_code_to_name
from degree.utils.degreeworks_client import DegreeworksClient
from degree.utils.parse_degreeworks import parse_and_save_degreeworks
Expand Down Expand Up @@ -51,7 +51,7 @@ def add_arguments(self, parser):
"--deduplicate-rules",
action="store_true",
)

parser.add_argument(
"--interactive",
action="store_true",
Expand Down Expand Up @@ -110,9 +110,11 @@ def handle(self, *args, **kwargs):
degree.save()
if kwargs["verbosity"]:
print(f"Saving degree {degree}...")
parse_and_save_degreeworks(client.audit(degree), degree, interactive=kwargs["interactive"])

if kwargs["deduplicate_rules"]:
parse_and_save_degreeworks(
client.audit(degree), degree, interactive=kwargs["interactive"]
)

if kwargs["deduplicate_rules"]:
if kwargs["verbosity"]:
print("Deduplicating rules...")
deduplicate_rules(verbose=kwargs["verbosity"])
deduplicate_rules(verbose=kwargs["verbosity"])
13 changes: 8 additions & 5 deletions backend/degree/management/commands/load_degrees.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@
from django.core.management.base import BaseCommand
from django.db import transaction

from degree.management.commands.deduplicate_rules import deduplicate_rules
from degree.models import Degree, program_code_to_name
from degree.utils.parse_degreeworks import parse_and_save_degreeworks
from degree.management.commands.deduplicate_rules import deduplicate_rules


class Command(BaseCommand):
help = dedent(
Expand Down Expand Up @@ -58,9 +59,12 @@ def handle(self, *args, **kwargs):
).groups()
if program not in program_code_to_name:
if kwargs["verbosity"]:
print(f"Skipping {degree_file} because {program} is not an applicable program code")
print(
f"Skipping {degree_file} because {program}"
"is not an applicable program code"
)
continue

if kwargs["verbosity"]:
print("Loading", degree_file, "...")

Expand Down Expand Up @@ -89,8 +93,7 @@ def handle(self, *args, **kwargs):
print(f"Parsing and saving degree {degree}...")
parse_and_save_degreeworks(degree_json, degree, interactive=kwargs["interactive"])

if kwargs["deduplicate_rules"]:
if kwargs["deduplicate_rules"]:
if kwargs["verbosity"]:
print("Deduplicating rules...")
deduplicate_rules(verbose=kwargs["verbosity"])

4 changes: 3 additions & 1 deletion backend/degree/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ class Fulfillment(models.Model):
related_name="+",
help_text=dedent(
"""
The last offering of the course with the full code, or null if
The last offering of the course with the full code, or null if
there is no such historical course.
"""
),
Expand Down Expand Up @@ -383,6 +383,8 @@ def update_satisfaction_statuses(sender, instance, action, pk_set, **kwargs):
[fulfillment.full_code for fulfillment in degree_plan.fulfillments.all()]
)
status.save()


m2m_changed.connect(update_satisfaction_statuses, sender=Fulfillment.rules.through)


Expand Down
6 changes: 5 additions & 1 deletion backend/degree/serializers.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,8 @@
from courses.serializers import CourseListSerializer
from degree.models import Degree, DegreePlan, DoubleCountRestriction, Fulfillment, Rule

class DegreeListSerializer(serializers.ModelSerializer):

class DegreeListSerializer(serializers.ModelSerializer):
class Meta:
model = Degree
fields = "__all__"
Expand All @@ -18,6 +18,8 @@ class RuleSerializer(serializers.ModelSerializer):
class Meta:
model = Rule
fields = "__all__"


# Allow recursive serialization of rules
RuleSerializer._declared_fields["rules"] = RuleSerializer(
many=True, read_only=True, source="children"
Expand All @@ -29,6 +31,7 @@ class Meta:
model = DoubleCountRestriction
fields = "__all__"


class DegreeDetailSerializer(serializers.ModelSerializer):
rules = RuleSerializer(many=True, read_only=True)
double_count_restrictions = DoubleCountRestrictionSerializer(many=True, read_only=True)
Expand All @@ -37,6 +40,7 @@ class Meta:
model = Degree
fields = "__all__"


class FulfillmentSerializer(serializers.ModelSerializer):
course = CourseListSerializer(
read_only=True,
Expand Down
7 changes: 1 addition & 6 deletions backend/degree/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@
from rest_framework.routers import DefaultRouter
from rest_framework_nested.routers import NestedDefaultRouter

from degree.views import (
DegreeViewset,
DegreePlanViewset,
FulfillmentViewSet,
courses_for_rule,
)
from degree.views import DegreePlanViewset, DegreeViewset, FulfillmentViewSet, courses_for_rule


router = DefaultRouter()
Expand Down
Loading

0 comments on commit d390f17

Please sign in to comment.