Merge branch 'master' into django-storages-upgrade

.eslintrc.json

@@ -27,6 +27,7 @@
     "func-names": "off",
     "indent": ["error", 4],
     "react/jsx-indent": ["error", 4],
+    "react/jsx-indent-props": ["error", 4],
     "new-cap": "off",
     "no-else-return": "off",
     "no-shadow": "error",
@@ -70,7 +71,6 @@
     "prefer-rest-params": "off",
     "prefer-template": "off",
     "radix": "off",
-    "react/jsx-indent-props": ["error", 4],
     "react/prop-types": "off",
     "vars-on-top": "off"
   }

.github/CODEOWNERS

@@ -43,9 +43,9 @@ openedx/features/content_type_gating/
 openedx/features/course_duration_limits/
 openedx/features/discounts/
 
-# Ping tCRIL On-call if someone uses the QuickStart
+# Ping Axim On-call if someone uses the QuickStart
 # https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html
-lms/templates/dashboard.html               @openedx/tcril-oncall
+lms/templates/dashboard.html               @openedx/axim-oncall
 
 # Ensure minimal.yml stays minimal, this could be a team in the future
 # but it's just me for now, others can sign up if they care as well.

.github/workflows/docker-compose.yml.mysqldbdump

@@ -16,7 +16,7 @@ services:
       retries: 10
   edxapp:
     image: edxops/edxapp:latest
-    command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && paver update_db'
+    command: bash -c 'source /edx/app/edxapp/edxapp_env && cd /edx/app/edxapp/edx-platform/ && make migrate'
     volumes:
        - ../../:/edx/app/edxapp/edx-platform
     depends_on:

.github/workflows/docker-publish.yml

@@ -9,14 +9,35 @@ jobs:
   # See also https://docs.docker.com/docker-hub/builds/
   push:
     runs-on: ubuntu-latest
-    if: github.event_name == 'push'
+    if: github.event_name == 'push'     
+
+    strategy:
+      matrix:
+        variant:
+          - "lms_dev"
+          - "cms_dev"
+          - "cms"
+          - "lms"
 
     steps:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Build and Push docker image
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+           username: ${{ secrets.DOCKERHUB_USERNAME }}
+           password: ${{ secrets.DOCKERHUB_PASSWORD }}
+
+      - name: Build and push lms base docker image
         env:
           DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
-          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
-        run : make docker_push
+          DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}                
+        run : make docker_tag_build_push_${{matrix.variant}}
+

.github/workflows/pylint-checks.yml

@@ -14,7 +14,7 @@ jobs:
       matrix:
         include:
           - module-name: lms-1
-            path: "--django-settings-module=lms.envs.test lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/ lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/ lms/djangoapps/courseware/ lms/djangoapps/course_goals/ lms/djangoapps/rss_proxy/ lms/djangoapps/save_for_later/"
+            path: "--django-settings-module=lms.envs.test lms/djangoapps/badges/ lms/djangoapps/branding/ lms/djangoapps/bulk_email/ lms/djangoapps/bulk_enroll/ lms/djangoapps/bulk_user_retirement/ lms/djangoapps/ccx/ lms/djangoapps/certificates/ lms/djangoapps/commerce/ lms/djangoapps/course_api/ lms/djangoapps/course_blocks/ lms/djangoapps/course_home_api/ lms/djangoapps/course_wiki/ lms/djangoapps/coursewarehistoryextended/ lms/djangoapps/debug/ lms/djangoapps/courseware/ lms/djangoapps/course_goals/ lms/djangoapps/rss_proxy/"
           - module-name: lms-2
             path: "--django-settings-module=lms.envs.test lms/djangoapps/gating/ lms/djangoapps/grades/ lms/djangoapps/instructor/ lms/djangoapps/instructor_analytics/ lms/djangoapps/discussion/ lms/djangoapps/edxnotes/ lms/djangoapps/email_marketing/ lms/djangoapps/experiments/ lms/djangoapps/instructor_task/ lms/djangoapps/learner_dashboard/ lms/djangoapps/learner_recommendations/ lms/djangoapps/learner_home/ lms/djangoapps/lms_initialization/ lms/djangoapps/lms_xblock/ lms/djangoapps/lti_provider/ lms/djangoapps/mailing/ lms/djangoapps/mobile_api/ lms/djangoapps/monitoring/ lms/djangoapps/ora_staff_grader/ lms/djangoapps/program_enrollments/ lms/djangoapps/rss_proxy lms/djangoapps/static_template_view/ lms/djangoapps/staticbook/ lms/djangoapps/support/ lms/djangoapps/survey/ lms/djangoapps/teams/ lms/djangoapps/tests/ lms/djangoapps/user_tours/ lms/djangoapps/verify_student/ lms/djangoapps/mfe_config_api/ lms/envs/ lms/lib/ lms/tests.py"
           - module-name: openedx-1

.github/workflows/unit-test-shards.json

@@ -64,7 +64,6 @@
       "lms/djangoapps/ora_staff_grader/",
       "lms/djangoapps/program_enrollments/",
       "lms/djangoapps/rss_proxy/",
-      "lms/djangoapps/save_for_later/",
       "lms/djangoapps/static_template_view/",
       "lms/djangoapps/staticbook/",
       "lms/djangoapps/support/",

.github/workflows/upgrade-one-python-dependency.yml

@@ -12,6 +12,10 @@ on:
          description: 'Name of package to upgrade'
          required: true
          type: string
+       version:
+          description: 'Version number to upgrade to in constraints.txt (only needed if pinned)'
+          default: ''
+          type: string
        change_desc:
          description: |
            Description of change, for commit message and PR. (What does the new version add or fix?)
@@ -37,6 +41,13 @@ jobs:
         with:
           python-version: "3.8"
 
+      - name: Update any pinned dependencies
+        env:
+          NEW_VERSION: "${{ inputs.version }}"
+          PACKAGE: "${{ inputs.package }}"
+        run: |
+          sed 's/^\('$PACKAGE'[^#]*\)==[^ #]\+/\1=='$NEW_VERSION'/' -i requirements/constraints.txt
+
       - name: Run make upgrade-package
         env:
           PACKAGE: "${{ inputs.package }}"
@@ -84,6 +95,7 @@ jobs:
           body: |
             ${{ env.body_prefix }}PR generated by workflow `${{ github.workflow_ref }}` on behalf of @${{ github.triggering_actor }}.
           assignees: "${{ github.triggering_actor }}"
+          reviewers: "${{ github.triggering_actor }}"
 
       - name: Job summary
         env:

.readthedocs.yaml

@@ -5,6 +5,11 @@ build:
   tools:
     python: "3.8"
 
+sphinx:
+  configuration: source/conf.py
+
 python:
   install:
   - requirements: "requirements/edx/doc.txt"
+  - method: pip
+    path: .

Dockerfile

@@ -57,6 +57,7 @@ RUN apt-get update && \
         libpython3.8 \
         libpython3.8-stdlib \
         libmysqlclient21 \
+        pkg-config \
         libssl1.1 \
         libxmlsec1-openssl \
         # lynx: Required by https://github.com/openedx/edx-platform/blob/b489a4ecb122/openedx/core/lib/html_to_text.py#L16
@@ -114,12 +115,19 @@ COPY requirements requirements
 RUN pip install -r requirements/pip.txt
 RUN pip install -r requirements/edx/base.txt
 
-# Install node and node modules
+# Install node and npm
 RUN nodeenv /edx/app/edxapp/nodeenv --node=16.14.0 --prebuilt
 RUN npm install -g [email protected]
+
+# This script is used by an npm post-install hook.
+# We copy it into the image now so that it will be available when we run `npm install` in the next step.
+# The script itself will copy certain modules into some uber-legacy parts of edx-platform which still use RequireJS.
+COPY scripts/copy-node-modules.sh scripts/copy-node-modules.sh
+
+# Install node modules
 COPY package.json package.json
 COPY package-lock.json package-lock.json
-RUN npm set progress=false && npm install
+RUN npm set progress=false && npm ci
 
 # The builder-development stage is a temporary stage that installs python modules required for development purposes
 # The built artifacts from this stage are then copied to the development stage.

Makefile

@@ -1,9 +1,10 @@
 # Do things in edx-platform
-.PHONY: api-docs-sphinx api-docs base-requirements check-types clean \
+.PHONY: base-requirements check-types clean \
   compile-requirements detect_changed_source_translations dev-requirements \
-  docker_auth docker_build docker_push docker_tag docs extract_translations \
-  guides help lint-imports local-requirements pre-requirements pull \
-  pull_translations push_translations requirements shell swagger \
+  docker_auth docker_build docker_tag_build_push_lms docker_tag_build_push_lms_dev \
+  docker_tag_build_push_cms docker_tag_build_push_cms_dev docs extract_translations \
+  guides help lint-imports local-requirements migrate migrate-lms migrate-cms \
+  pre-requirements pull pull_translations push_translations requirements shell swagger \
   technical-docs test-requirements ubuntu-requirements upgrade-package upgrade
 
 # Careful with mktemp syntax: it has to work on Mac and Ubuntu, which have differences.
@@ -23,24 +24,17 @@ clean: ## archive and delete most git-ignored files
 	tar xf $(PRIVATE_FILES)
 	rm $(PRIVATE_FILES)
 
-SWAGGER = docs/swagger.yaml
+SWAGGER = docs/lms-openapi.yaml
 
-docs: api-docs guides technical-docs ## build all the developer documentation for this repository
+docs: guides technical-docs ## build all the developer documentation for this repository
 
 swagger: ## generate the swagger.yaml file
 	DJANGO_SETTINGS_MODULE=docs.docs_settings python manage.py lms generate_swagger --generator-class=edx_api_doc_tools.ApiSchemaGenerator -o $(SWAGGER)
 
-api-docs-sphinx: swagger	## generate the sphinx source files for api-docs
-	rm -f docs/api/gen/*
-	python docs/sw2sphinxopenapi.py $(SWAGGER) docs/api/gen
-
-api-docs: api-docs-sphinx	## build the REST api docs
-	cd docs/api; make html
-
 technical-docs:  ## build the technical docs
 	$(MAKE) -C docs/technical html
 
-guides:	## build the developer guide docs
+guides:	swagger ## build the developer guide docs
 	cd docs/guides; make clean html
 
 extract_translations: ## extract localizable strings from sources
@@ -101,12 +95,13 @@ shell: ## launch a bash shell in a Docker container with all edx-platform depend
 # Order is very important in this list: files must appear after everything they include!
 REQ_FILES = \
 	requirements/edx/coverage \
-	requirements/edx/doc \
 	requirements/edx/paver \
 	requirements/edx-sandbox/py38 \
 	requirements/edx/base \
+	requirements/edx/doc \
 	requirements/edx/testing \
 	requirements/edx/development \
+	requirements/edx/assets \
 	scripts/xblock/requirements
 
 define COMMON_CONSTRAINTS_TEMP_COMMENT
@@ -124,6 +119,8 @@ compile-requirements: pre-requirements $(COMMON_CONSTRAINTS_TXT) ## Re-compile *
 	@# Bootstrapping: Rebuild pip and pip-tools first, and then install them
 	@# so that if there are any failures we'll know now, rather than the next
 	@# time someone tries to use the outputs.
+	sed '/^django-simple-history==/d' requirements/common_constraints.txt > requirements/common_constraints.tmp
+	mv requirements/common_constraints.tmp requirements/common_constraints.txt
 	pip-compile -v --allow-unsafe ${COMPILE_OPTS} -o requirements/pip.txt requirements/pip.in
 	pip install -r requirements/pip.txt
 
@@ -149,34 +146,40 @@ upgrade-package: ## update just one package to the latest usable release
 check-types: ## run static type-checking tests
 	mypy
 
-docker_build:
+docker_auth:
+	echo "$$DOCKERHUB_PASSWORD" | docker login -u "$$DOCKERHUB_USERNAME" --password-stdin
+
+docker_build: docker_auth
 	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development -t openedx/lms-dev
 	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production -t openedx/lms
 	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development -t openedx/cms-dev
 	DOCKER_BUILDKIT=1 docker build . --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production -t openedx/cms
 
-docker_tag: docker_build
-	docker tag openedx/lms     openedx/lms:${GITHUB_SHA}
-	docker tag openedx/lms-dev openedx/lms-dev:${GITHUB_SHA}
-	docker tag openedx/cms     openedx/cms:${GITHUB_SHA}
-	docker tag openedx/cms-dev openedx/cms-dev:${GITHUB_SHA}
+docker_tag_build_push_lms: docker_auth
+	docker buildx build -t openedx/lms:latest -t openedx/lms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target production --push .
 
-docker_auth:
-	echo "$$DOCKERHUB_PASSWORD" | docker login -u "$$DOCKERHUB_USERNAME" --password-stdin
+docker_tag_build_push_lms_dev: docker_auth
+	docker buildx build -t openedx/lms-dev:latest -t openedx/lms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=lms --build-arg SERVICE_PORT=8000 --target development --push .
+
+docker_tag_build_push_cms: docker_auth
+	docker buildx build -t openedx/cms:latest -t openedx/cms:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target production --push .
 
-docker_push: docker_tag docker_auth ## push to docker hub
-	docker push "openedx/lms:latest"
-	docker push "openedx/lms:${GITHUB_SHA}"
-	docker push "openedx/lms-dev:latest"
-	docker push "openedx/lms-dev:${GITHUB_SHA}"
-	docker push "openedx/cms:latest"
-	docker push "openedx/cms:${GITHUB_SHA}"
-	docker push "openedx/cms-dev:latest"
-	docker push "openedx/cms-dev:${GITHUB_SHA}"
+docker_tag_build_push_cms_dev: docker_auth
+	docker buildx build -t openedx/cms-dev:latest -t openedx/cms-dev:${GITHUB_SHA} --platform linux/amd64,linux/arm64 --build-arg SERVICE_VARIANT=cms --build-arg SERVICE_PORT=8010 --target development --push .
 
 lint-imports:
 	lint-imports
 
+migrate-lms:
+	python manage.py lms showmigrations --database default --traceback --pythonpath=.
+	python manage.py lms migrate --database default --traceback --pythonpath=.
+
+migrate-cms:
+	python manage.py cms showmigrations --database default --traceback --pythonpath=.
+	python manage.py cms migrate --database default --noinput --traceback --pythonpath=.
+
+migrate: migrate-lms migrate-cms
+
 # WARNING (EXPERIMENTAL):
 # This installs the Ubuntu requirements necessary to make `pip install` and some other basic
 # dev commands to pass. This is not necessarily everything needed to get a working edx-platform.

cms/djangoapps/contentstore/api/urls.py

@@ -8,7 +8,6 @@
 
 
 app_name = 'contentstore'
-helper = "{0,1}"
 
 urlpatterns = [
     re_path(fr'^v0/import/{settings.COURSE_ID_PATTERN}/$',