diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 02731cb3b4e5..06acb35aae2e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -46,3 +46,7 @@ openedx/features/discounts/ # Ping tCRIL On-call if someone uses the QuickStart # https://docs.openedx.org/en/latest/developers/quickstarts/first_openedx_pr.html lms/templates/dashboard.html @openedx/tcril-oncall + +# Ensure minimal.yml stays minimal, this could be a team in the future +# but it's just me for now, others can sign up if they care as well. +lms/envs/minimal.yml @feanil diff --git a/.github/workflows/migrations-check-mysql8.yml b/.github/workflows/migrations-check-mysql8.yml deleted file mode 100644 index 74c4e6cabd84..000000000000 --- a/.github/workflows/migrations-check-mysql8.yml +++ /dev/null @@ -1,80 +0,0 @@ -name: Migrations check on MySql 8.0 - -on: - workflow_dispatch: - pull_request: - push: - branches: - - master - -jobs: - check_migrations: - name: check migrations mysql8 - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ ubuntu-20.04 ] - python-version: [ 3.8 ] - - steps: - - name: Checkout repo - uses: actions/checkout@v2 - - - name: Setup Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - - name: Install system Packages - run: | - sudo apt-get update - sudo apt-get install -y libxmlsec1-dev - - - name: Get pip cache dir - id: pip-cache-dir - run: | - echo "::set-output name=dir::$(pip cache dir)" - - - name: Cache pip dependencies - id: cache-dependencies - uses: actions/cache@v2 - with: - path: ${{ steps.pip-cache-dir.outputs.dir }} - key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} - restore-keys: ${{ runner.os }}-pip- - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - - name: Install Python dependencies - run: | - make dev-requirements - pip uninstall -y mysqlclient - pip install --no-binary mysqlclient mysqlclient - pip uninstall -y xmlsec - pip install --no-binary xmlsec xmlsec - - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat <- + --health-cmd "mongo --quiet --eval 'db.runCommand(\"ping\")'" + --health-interval 10s + --health-timeout 5s + --health-retries 3 + mysql: + image: mysql:${{ matrix.mysql-version }} + ports: + - 3306:3306 + env: + MYSQL_DATABASE: "edxapp" + MYSQL_USER: "edxapp001" + MYSQL_PASSWORD: "password" + MYSQL_RANDOM_ROOT_PASSWORD: true + options: >- + --health-cmd "mysqladmin ping" + --health-interval 10s + --health-timeout 5s + --health-retries 3 steps: + - name: Setup mongodb user + run: | + mongosh edxapp --eval ' + db.createUser( + { + user: "edxapp", + pwd: "password", + roles: [ + { role: "readWrite", db: "edxapp" }, + ] + } + ); + ' + + - name: Verify mongo and mysql db credentials + run: | + mysql -h 127.0.0.1 -uedxapp001 -ppassword -e "select 1;" edxapp + mongosh --host 127.0.0.1 --username edxapp --password password --eval 'use edxapp; db.adminCommand("ping");' edxapp + - name: Checkout repo uses: actions/checkout@v2 @@ -28,7 +74,7 @@ jobs: - name: Install system Packages run: | sudo apt-get update - sudo apt-get install -y libxmlsec1-dev + make ubuntu-requirements - name: Get pip cache dir id: pip-cache-dir @@ -43,51 +89,17 @@ jobs: key: ${{ runner.os }}-pip-${{ hashFiles('requirements/edx/development.txt') }} restore-keys: ${{ runner.os }}-pip- - - name: Install MySQL 5.7 - run: | - mkdir ${RUNNER_WORKSPACE}/mysql_packages && cd ${RUNNER_WORKSPACE}/mysql_packages - sudo apt-get remove --purge *mysql* -y - sudo rm -rvf /etc/init.d/mysql* /etc/mysql* /var/lib/mysql* - sudo apt-get install libaio1 libnuma1 libtinfo5 -y - wget https://cdn.mysql.com/archives/mysql-5.7/mysql-server_5.7.35-1ubuntu18.04_amd64.deb-bundle.tar - tar -xf mysql-server_5.7*.tar - sudo DEBIAN_FRONTEND=noninteractive dpkg --install mysql-common_5.7*.deb libmysqlclient*.deb mysql-server_5.7*.deb mysql-client_5.7*.deb mysql-community-server_5.7*.deb mysql-community-client_5.7*.deb - - - name: Ubuntu and sql Versions - run: | - lsb_release -a - mysql -V - - name: Install Python dependencies run: | make dev-requirements - - name: Initiate Services - run: | - sudo systemctl start mongod - sudo /etc/init.d/mysql start - - - name: Reset mysql password - run: | - cat <`_ is a service-oriented platform for authoring and delivering online learning at any scale. The platform is written in Python and JavaScript and makes extensive use of the Django @@ -27,7 +26,7 @@ platform. Functionally, the edx-platform repository provides two services: * LMS (Learning Management Service), which delivers learning content. Installation ------------- +************ Installing and running an Open edX instance is not simple. We strongly recommend that you use a service provider to run the software for you. They @@ -43,8 +42,28 @@ so, `Open edX Installation Options`_ explains your options. .. _Open edX Developer Stack: https://github.com/openedx/devstack .. _Open edX Installation Options: https://openedx.atlassian.net/wiki/spaces/OpenOPS/pages/60227779/Open+edX+Installation+Options +Dependencies +============ + +In order to build and run this code you'll need the following available on your +system: + +Interperters/Tools: + +* Python 3.8 + +* Node 16 + +Services: + +* MySQL 5.7 + +* Mongo 4.x + +* Memcached + License -------- +******* The code in this repository is licensed under version 3 of the AGPL unless otherwise noted. Please see the `LICENSE`_ file for details. @@ -53,7 +72,7 @@ unless otherwise noted. Please see the `LICENSE`_ file for details. More about Open edX -------------------- +******************* See the `Open edX site`_ to learn more about the Open edX world. You can find information about hosting, extending, and contributing to Open edX software. In @@ -63,13 +82,13 @@ and other rich community resources. .. _Open edX site: https://openedx.org Documentation -------------- +************* -Documentation can be found at https://docs.edx.org. +Documentation can be found at https://docs.openedx.org. Getting Help ------------- +************ If you're having trouble, we have discussion forums at https://discuss.openedx.org where you can connect with others in the community. @@ -85,7 +104,7 @@ For more information about these options, see the `Getting Help`_ page. Issue Tracker -------------- +************* We use JIRA for our issue tracker, not GitHub issues. You can search `previously reported issues`_. If you need to report a problem, @@ -96,7 +115,7 @@ please make a free account on our JIRA and `create a new issue`_. How to Contribute ------------------ +***************** Contributions are welcome! The first step is to submit a signed `individual contributor agreement`_. See our `CONTRIBUTING`_ file for more @@ -105,7 +124,7 @@ quality, which will make your contribution more likely to be accepted. Reporting Security Issues -------------------------- +************************* Please do not report security issues in public. Please email security@edx.org. diff --git a/cms/envs/common.py b/cms/envs/common.py index b1b8c2cab05e..2425c190c2db 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -1052,7 +1052,7 @@ 'ATOMIC_REQUESTS': True, 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1062,7 +1062,7 @@ 'read_replica': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1072,7 +1072,7 @@ 'student_module_history': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp_csmh', 'OPTIONS': {}, 'PASSWORD': 'password', diff --git a/lms/envs/common.py b/lms/envs/common.py index 6c1dd7e42077..cd67373852d4 100644 --- a/lms/envs/common.py +++ b/lms/envs/common.py @@ -1613,7 +1613,7 @@ def _make_mako_template_dirs(settings): 'DOC_STORE_CONFIG': DOC_STORE_CONFIG, 'OPTIONS': { 'default_class': 'xmodule.hidden_block.HiddenBlock', - 'fs_root': DATA_DIR, + 'fs_root': lambda settings: settings.DATA_DIR, 'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string', } }, @@ -1623,7 +1623,7 @@ def _make_mako_template_dirs(settings): 'DOC_STORE_CONFIG': DOC_STORE_CONFIG, 'OPTIONS': { 'default_class': 'xmodule.hidden_block.HiddenBlock', - 'fs_root': DATA_DIR, + 'fs_root': lambda settings: settings.DATA_DIR, 'render_template': 'common.djangoapps.edxmako.shortcuts.render_to_string', } } @@ -1632,6 +1632,7 @@ def _make_mako_template_dirs(settings): } } + DATABASES = { # edxapp's edxapp-migrate scripts and the edxapp_migrate play # will ensure that any DB not named read_replica will be migrated @@ -1640,7 +1641,7 @@ def _make_mako_template_dirs(settings): 'ATOMIC_REQUESTS': True, 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1650,7 +1651,7 @@ def _make_mako_template_dirs(settings): 'read_replica': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp', 'OPTIONS': {}, 'PASSWORD': 'password', @@ -1660,7 +1661,7 @@ def _make_mako_template_dirs(settings): 'student_module_history': { 'CONN_MAX_AGE': 0, 'ENGINE': 'django.db.backends.mysql', - 'HOST': 'localhost', + 'HOST': '127.0.0.1', 'NAME': 'edxapp_csmh', 'OPTIONS': {}, 'PASSWORD': 'password', diff --git a/lms/envs/minimal.yml b/lms/envs/minimal.yml index 54c3bcea2610..0688b7c49c3a 100644 --- a/lms/envs/minimal.yml +++ b/lms/envs/minimal.yml @@ -20,6 +20,16 @@ MKTG_URL_LINK_MAP: {} MKTG_URL_OVERRIDES: {} REST_FRAMEWORK: {} +# We need MEDIA_ROOT because otherwise ./lms/djangoapps/certificates/migrations/0003_data__default_modes.py will fail. +# +# Using a local tmp location is not a good default for a production system. For production we should use +# off-machine storage to more easily scale the system. +MEDIA_ROOT: "/tmp/edx-platform/media/" + +# DATA_DIR is overridden twice in ./lms/envs/common.py override it here temporarily until that's cleaned up and we +# can default to something dev friendly and have overrides in production.py for production friendly settings. +DATA_DIR: "/tmp/edx-platform/data_dir" + # For just the CMS LMS_ROOT_URL: "http://localhost" LMS_INTERNAL_ROOT_URL: "http://localhost" diff --git a/lms/envs/production.py b/lms/envs/production.py index 342877206b0b..c8587f8fa6d7 100644 --- a/lms/envs/production.py +++ b/lms/envs/production.py @@ -490,6 +490,18 @@ def get_env_setting(setting): # Get the MODULESTORE from auth.json, but if it doesn't exist, # use the one from common.py MODULESTORE = convert_module_store_setting_if_needed(AUTH_TOKENS.get('MODULESTORE', MODULESTORE)) + +# After conversion above, the modulestore will have a "stores" list with all defined stores, for all stores, add the +# fs_root entry to derived collection so that if it's a callable it can be resolved. We need to do this because the +# `derived_collection_entry` takes an exact index value but the config file might have overidden the number of stores +# and so we can't be sure that the 2 we define in common.py will be there when we try to derive settings. This could +# lead to execptions being thrown when the `derive_settings` call later in this file tries to update settings. We call +# the derived_collection_entry function here to ensure that we update the fs_root for any callables that remain after +# we've updated the MODULESTORE setting from our config file. +for idx, store in enumerate(MODULESTORE['default']['OPTIONS']['stores']): + if 'OPTIONS' in store and 'fs_root' in store["OPTIONS"]: + derived_collection_entry('MODULESTORE', 'default', 'OPTIONS', 'stores', idx, 'OPTIONS', 'fs_root') + MONGODB_LOG = AUTH_TOKENS.get('MONGODB_LOG', {}) EMAIL_HOST_USER = AUTH_TOKENS.get('EMAIL_HOST_USER', '') # django default is ''