diff --git a/Dockerfile b/Dockerfile index e77dd47473a8..c6518fe175fa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -117,6 +117,7 @@ RUN pip install -r requirements/edx/base.txt # Install node and node modules RUN nodeenv /edx/app/edxapp/nodeenv --node=16.14.0 --prebuilt +COPY scripts/copy-node-modules.sh scripts RUN npm install -g npm@8.5.x COPY package.json package.json COPY package-lock.json package-lock.json diff --git a/docs/decisions/0017-reimplement-asset-processing.rst b/docs/decisions/0017-reimplement-asset-processing.rst index 7f16dcc249ff..5eb4ecafde94 100644 --- a/docs/decisions/0017-reimplement-asset-processing.rst +++ b/docs/decisions/0017-reimplement-asset-processing.rst @@ -128,20 +128,22 @@ The three top-level edx-platform asset processing actions are *build*, *collect* A Python-defined task that calls out to each build stage. - - ``scripts/build-assets.sh`` + - ``npm clean-install && npm run build`` + + Simple NPM wrappers around the build stages. The wrappers will be written in Bash and tested on both GNU+Linux and macOS. + + These commands are a "one stop shop" for building assets, but more efficiency-oriented users may choose to run build stages individually. - A Bash script that contains all build stages, with subcommands available for running each stage separately. Its command-line interface inspired by Tutor's ``openedx-assets`` script. The script will be runnable on any POSIX system, including macOS and Ubuntu and it will linted for common shell scripting mistakes using `shellcheck `_. - * - + **Build stage 1: Copy npm-installed assets** from node_modules to other folders in edx-platform. They are used by certain especially-old legacy LMS & CMS frontends that are not set up to work with npm directly. - ``paver update_assets --skip-collect`` Implemented in Python within update_assets. There is no standalone command for it. - - ``scripts/build-assets.sh npm`` + - ``npm install`` + + An NPM post-install hook will automatically call scripts/copy-node-modules.sh, a pure Bash reimplementation of the node_modules asset copying, whenever ``npm install`` is invoked. - Pure Bash reimplementation. See *Rejected Alternatives* for a note about this. - * - + **Build stage 2: Copy XModule fragments** from the xmodule source tree over to input directories for Webpack and SCSS compilation. This is required for a hard-coded list of old XModule-style XBlocks. This is not required for new pure XBlocks, which include (or pip-install) their assets into edx-platform as ready-to-serve JS/CSS/etc fragments. - ``paver process_xmodule_assets``, or @@ -150,25 +152,22 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Equivalent paver task and console script, both pointing at to an application-level Python module. That module inspects attributes from legacy XModule-style XBlock classes in order to determine which static assets to copy and what to name them. - - ``scripts/build-assets.sh xmodule`` + - (step no longer needed) - Initially, this command will just call out to the existing ``xmodule_assets`` command. Eventually, in order to make this step Python-free, we will need do either one or both of the following: + We will `remove the need for this step entirely `_. - + `Reimplement this step in Bash `_. - + `Remove the need for this step entirely `_. - * - + **Build stage 3: Run Webpack** in order to to shim, minify, otherwise process, and bundle JS modules. This requires a call to the npm-installed ``webpack`` binary. - ``paver webpack`` Python wrapper around a call to webpack. Invokes the ``./manage.py [lms|cms] print_setting`` multiple times in order to determine Django settings, adding which can add 20+ seconds to the build. - - ``scripts/build-assets.sh webpack --static-root "$(./manage.py lms print_setting STATIC_ROOT)"``. + - ``npm run webpack`` or ``npm run webpack-dev`` - Bash wrapper around a call to webpack. The script will accept parameters rather than looking up Django settings itself. + Simple shell script defined in package.json to invoke Webpack in prod or dev mode. The script will look for several environment variables, with a default defined for each one. See **Build Configuration** for details. The script will NOT invoke ``print_setting``; we leave to distributions the tasking of setting environment variables appropriately. + + To continue using ``print_setting``, one could run: ``STATIC_ROOT_LMS="$(./manage.py lms print_setting STATIC_ROOT_LMS)" npm run webpack`` - The print_setting command will still be available for distributions to use to extract ``STATIC_ROOT`` from Django settings, but it will only need to be run once. As described in **Build Configuration** below, unnecessary Django settings will be removed. Some distributions may not even need to look up ``STATIC_ROOT``; Tutor, for example, will probably render ``STATIC_ROOT`` directly into the environment variable ``OPENEDX_BUILD_ASSETS_OPTS`` variable, described in the **Build Configuration**. - * - + **Build stage 4: Compile default SCSS** into CSS for legacy LMS/CMS frontends. - ``paver compile_sass`` @@ -177,28 +176,26 @@ The three top-level edx-platform asset processing actions are *build*, *collect* Note: libsass is pinned to a 2015 version with a non-trivial upgrade path. Installing it requires compiling a large C extension, noticeably affecting Docker image build time. - - ``scripts/build-assets.sh css`` + - ``scripts/compile_sass`` + + TODO describe - Bash reimplementation, calling ``node-sass`` and ``rtlcss``. - - The initial implementation of build-assets.sh may use ``sassc``, a CLI provided by libsass, instead of node-sass. Then, ``sassc`` can be replaced by ``node-sass`` as part of a subsequent `edx-platform frontend framework upgrade effort `_. + TODO mention `edx-platform frontend framework upgrade effort `_. * - + **Build stage 5: Compile themes' SCSS** into CSS for legacy LMS/CMS frontends. The default SCSS is used as a base, and theme-provided SCSS files are used as overrides. Themes are searched for from some number of operator-specified theme directories. - ``./manage.py [lms|cms] compile_sass``, or - ``paver compile_sass --theme-dirs ...`` + ``paver compile_sass --theme-dirs X Y --themes A B`` The management command is a wrapper around the paver task. The former looks up the list of theme search directories from Django settings and site configuration; the latter requires them to be supplied as arguments. - ``./manage.py [lms|cms] compile_sass``, or - ``scripts/build-assets.sh themes --theme-dirs ...`` + - ``scripts/compile_sass --theme-dir X --theme-dir Y --theme A --theme B`` - The management command will remain available, but it will need to be updated to point at the Bash script, which will replace the paver task (see build stage 4 for details). + The management command will remain available, but it will need to be updated to point at ``compile_sass``, which will replace the paver task (see build stage 4 for details). - The overall asset *build* action will use the Bash script; this means that list of theme directories will need to be provided as arguments, but it ensures that the build can remain Python-free. - * - **Collect** the built static assets from edx-platform to another location (the ``STATIC_ROOT``) so that they can be efficiently served *without* Django's webserver. This step, by nature, requires Python and Django in order to find and organize the assets, which may come from edx-platform itself or from its many installed Python and NPM packages. This is only needed for **production** environments, where it is usually desirable to serve assets with something efficient like NGINX. - ``paver update_assets`` @@ -210,7 +207,7 @@ The three top-level edx-platform asset processing actions are *build*, *collect* - ``./manage.py lms collectstatic --noinput && ./manage.py cms collectstatic --noinput`` The standard Django interface will be used without a wrapper. The ignore patterns will be added to edx-platform's `staticfiles app configuration `_ so that they do not need to be supplied as part of the command. - + * - **Watch** static assets for changes in the background. When a change occurs, rebuild them automatically, so that the Django webserver picks up the changes. This is only necessary in **development** environments. A few different sets of assets may be watched: XModule fragments, Webpack assets, default SCSS, and theme SCSS. - ``paver watch_assets`` @@ -264,6 +261,18 @@ Furthermore, to facilitate a Python-free build reimplementation, we will remove OPENEDX_BUILD_ASSETS_OPTS=\ '--webpack-config path/to/webpack.my.config.js' + * - STATIC_ROOT_LMS + + - TODO + + - TODO + + * - STATIC_ROOT_CMS + + - TODO + + - TODO + * - JS_ENV_EXTRA_CONFIG - Global configuration object available to edx-platform JS modules. Defaults empty. Only known use is to add configuration and plugins for the TinyMCE editor. @@ -300,13 +309,15 @@ Either way, the migration path is straightforward: * - ``openedx-assets build`` - ``scripts/build-assets.sh`` * - ``openedx-assets npm`` - - ``scripts/build-assets.sh npm`` + - ``scripts/copy-node-modules.sh # (automatically invoked by 'npm install'!) * - ``openedx-assets xmodule`` - - ``scripts/build-assets.sh xmodule`` + - (no longer needed) * - ``openedx-assets common`` - - ``scripts/build-assets.sh css`` + - ``compile_sass --skip-themes`` * - ``openedx-assets themes`` - - ``scripts/build-assets.sh themes`` + - ``compile_sass --skip-default`` + * - ``openedx-assets webpack [--env=dev]`` + - ``npm run webpack[-dev]`` * - ``openedx-assets collect`` - ``./manage.py [lms|cms] collectstatic --noinput`` * - ``openedx-assets watch-themes`` @@ -328,13 +339,6 @@ OpenCraft has also performed a discovery on a `modernized system for static asse Rejected Alternatives ********************* -Copy node_modules via npm post-install -====================================== - -It was noted that `npm supports lifecycle scripts `_ in package.json, including ``postinstall``. We could use a post-install script to copy assets out of node_modules; this would occurr automatically after ``npm install``. Arguably, this would be more idiomatic than this ADR's proposal of ``scripts/build-assets.sh npm``. - -For now, we decided against this. While it seems like a good potential future improvement, we are currently unsure how it would interact with `moving node_modules out of edx-platform in Tutor `_, which is a motivation behind this ADR. For example, if node_modules could be located anywhere on the image, then we are not sure how the post-install script could know its target directory without us hard-coding Tutor's directory structure into the script. - Live with the problem ====================== diff --git a/lms/djangoapps/courseware/tests/test_discussion_xblock.py b/lms/djangoapps/courseware/tests/test_discussion_xblock.py index ea265daab764..ba176cc16eb2 100644 --- a/lms/djangoapps/courseware/tests/test_discussion_xblock.py +++ b/lms/djangoapps/courseware/tests/test_discussion_xblock.py @@ -218,7 +218,7 @@ def test_has_permission(self): """ permission_canary = object() with mock.patch( - 'lms.djangoapps.discussion.django_comment_client.permissions.has_permission', + 'xmodule.discussion_block.has_permission', return_value=permission_canary, ) as has_perm: actual_permission = self.block.has_permission("test_permission") diff --git a/package.json b/package.json index fb855d32a3e9..d8385ce8344b 100644 --- a/package.json +++ b/package.json @@ -2,6 +2,15 @@ "name": "edx", "version": "0.1.0", "repository": "https://github.com/openedx/edx-platform", + "scripts": { + "postinstall": "scripts/copy-node-modules.sh", + "build": "npm run webpack && npm run compile-sass", + "build-dev": "npm run webpack-dev && npm run compile-sass-dev", + "webpack": "NODE_ENV=${NODE_ENV:-production} \"$(npm bin)/webpack\" --progress --config=${WEBPACK_CONFIG_PATH:-webpack.prod.config.js}", + "webpack-dev": "NODE_ENV=development WEBPACK_CONFIG_PATH=webpack.dev.config.js npm run webpack", + "compile-sass": "compile_sass", + "compile-sass-dev": "compile_sass --env=dev" + }, "dependencies": { "@babel/core": "7.19.0", "@babel/plugin-proposal-object-rest-spread": "^7.18.9", diff --git a/pavelib/assets.py b/pavelib/assets.py index 3bf8a9ce88ae..8f950fe3f647 100644 --- a/pavelib/assets.py +++ b/pavelib/assets.py @@ -46,39 +46,6 @@ path('node_modules'), ] -# A list of NPM installed libraries that should be copied into the common -# static directory. -# If string ends with '/' then all file in the directory will be copied. -NPM_INSTALLED_LIBRARIES = [ - 'backbone.paginator/lib/backbone.paginator.js', - 'backbone/backbone.js', - 'bootstrap/dist/js/bootstrap.bundle.js', - 'hls.js/dist/hls.js', - 'jquery-migrate/dist/jquery-migrate.js', - 'jquery.scrollto/jquery.scrollTo.js', - 'jquery/dist/jquery.js', - 'moment-timezone/builds/moment-timezone-with-data.js', - 'moment/min/moment-with-locales.js', - 'picturefill/dist/picturefill.js', - 'requirejs/require.js', - 'underscore.string/dist/underscore.string.js', - 'underscore/underscore.js', - '@edx/studio-frontend/dist/', - 'which-country/index.js' -] - -# A list of NPM installed developer libraries that should be copied into the common -# static directory only in development mode. -NPM_INSTALLED_DEVELOPER_LIBRARIES = [ - 'sinon/pkg/sinon.js', - 'squirejs/src/Squire.js', -] - -# Directory to install static vendor files -NPM_JS_VENDOR_DIRECTORY = path('common/static/common/js/vendor') -NPM_CSS_VENDOR_DIRECTORY = path("common/static/common/css/vendor") -NPM_CSS_DIRECTORY = path("common/static/common/css") - # system specific lookup path additions, add sass dirs if one system depends on the sass files for other systems SASS_LOOKUP_DEPENDENCIES = { 'cms': [path('lms') / 'static' / 'sass' / 'partials', ], @@ -356,7 +323,6 @@ class SassWatcher(PatternMatchingEventHandler): """ ignore_directories = True patterns = ['*.scss'] - ignore_patterns = ['common/static/xmodule/*'] def register(self, observer, directories): """ @@ -385,47 +351,6 @@ def on_any_event(self, event): traceback.print_exc() -class XModuleSassWatcher(SassWatcher): - """ - Watches for sass file changes - """ - ignore_directories = True - ignore_patterns = [] - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - process_xmodule_assets() - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - -class XModuleAssetsWatcher(PatternMatchingEventHandler): - """ - Watches for css and js file changes - """ - ignore_directories = True - patterns = ['*.css', '*.js'] - - def register(self, observer): - """ - Register files with observer - """ - observer.schedule(self, 'xmodule/', recursive=True) - - @debounce() - def on_any_event(self, event): - print('\tCHANGED:', event.src_path) - try: - process_xmodule_assets() - except Exception: # pylint: disable=broad-except - traceback.print_exc() - - # To refresh the hash values of static xmodule content - restart_django_servers() - - @task @no_help @cmdopts([ @@ -644,73 +569,17 @@ def process_npm_assets(): """ Process vendor libraries installed via NPM. """ - def copy_vendor_library(library, skip_if_missing=False): - """ - Copies a vendor library to the shared vendor directory. - """ - if library.startswith('node_modules/'): - library_path = library - else: - library_path = f'node_modules/{library}' - - if library.endswith('.css') or library.endswith('.css.map'): - vendor_dir = NPM_CSS_VENDOR_DIRECTORY - else: - vendor_dir = NPM_JS_VENDOR_DIRECTORY - if os.path.exists(library_path): - sh('/bin/cp -rf {library_path} {vendor_dir}'.format( - library_path=library_path, - vendor_dir=vendor_dir, - )) - elif not skip_if_missing: - raise Exception(f'Missing vendor file {library_path}') - - def copy_vendor_library_dir(library_dir, skip_if_missing=False): - """ - Copies all vendor libraries in directory to the shared vendor directory. - """ - library_dir_path = f'node_modules/{library_dir}' - print(f'Copying vendor library dir: {library_dir_path}') - if os.path.exists(library_dir_path): - for dirpath, _, filenames in os.walk(library_dir_path): - for filename in filenames: - copy_vendor_library(os.path.join(dirpath, filename), skip_if_missing=skip_if_missing) - - # Skip processing of the libraries if this is just a dry run - if tasks.environment.dry_run: - tasks.environment.info("install npm_assets") - return - - # Ensure that the vendor directory exists - NPM_JS_VENDOR_DIRECTORY.mkdir_p() - NPM_CSS_DIRECTORY.mkdir_p() - NPM_CSS_VENDOR_DIRECTORY.mkdir_p() - - # Copy each file to the vendor directory, overwriting any existing file. - print("Copying vendor files into static directory") - for library in NPM_INSTALLED_LIBRARIES: - if library.endswith('/'): - copy_vendor_library_dir(library) - else: - copy_vendor_library(library) - - # Copy over each developer library too if they have been installed - print("Copying developer vendor files into static directory") - for library in NPM_INSTALLED_DEVELOPER_LIBRARIES: - copy_vendor_library(library, skip_if_missing=True) + sh('scripts/copy-node-modules.sh') @task -@needs( - 'pavelib.prereqs.install_python_prereqs', -) @no_help def process_xmodule_assets(): """ Process XModule static assets. """ - sh('xmodule_assets common/static/xmodule') - print("\t\tFinished processing xmodule assets.") + print("\t\tProcessing xmodule assets is no longer needed. This task is now a no-op.") + print("\t\tWhen paver is removed from edx-platform, this step will not replaced.") def restart_django_servers(): @@ -908,8 +777,6 @@ def watch_assets(options): observer = Observer(timeout=wait) SassWatcher().register(observer, sass_directories) - XModuleSassWatcher().register(observer, ['xmodule/']) - XModuleAssetsWatcher().register(observer) print("Starting asset watcher...") observer.start() @@ -931,6 +798,7 @@ def watch_assets(options): @task @needs( 'pavelib.prereqs.install_node_prereqs', + 'pavelib.prereqs.install_python_prereqs', ) @consume_args @timed @@ -982,9 +850,6 @@ def update_assets(args): args = parser.parse_args(args) collect_log_args = {} - process_xmodule_assets() - process_npm_assets() - # Build Webpack call_task('pavelib.assets.webpack', options={'settings': args.settings}) diff --git a/pavelib/js_test.py b/pavelib/js_test.py index fed1f0fe102f..3f84a3361dba 100644 --- a/pavelib/js_test.py +++ b/pavelib/js_test.py @@ -26,7 +26,6 @@ @needs( 'pavelib.prereqs.install_node_prereqs', 'pavelib.utils.test.utils.clean_reports_dir', - 'pavelib.assets.process_xmodule_assets', ) @cmdopts([ ("suite=", "s", "Test suite to run"), diff --git a/pavelib/paver_tests/test_servers.py b/pavelib/paver_tests/test_servers.py index af60d247de29..9c2a3425fb32 100644 --- a/pavelib/paver_tests/test_servers.py +++ b/pavelib/paver_tests/test_servers.py @@ -230,7 +230,6 @@ def verify_server_task(self, task_name, options): expected_asset_settings = "test_static_optimized" expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS if not is_fast: - expected_messages.append("xmodule_assets common/static/xmodule") expected_messages.append("install npm_assets") expected_messages.extend( [c.format(settings=expected_asset_settings, @@ -275,7 +274,6 @@ def verify_run_all_servers_task(self, options): expected_collect_static = not is_fast and expected_settings != Env.DEVSTACK_SETTINGS expected_messages = [] if not is_fast: - expected_messages.append("xmodule_assets common/static/xmodule") expected_messages.append("install npm_assets") expected_messages.extend( [c.format(settings=expected_asset_settings, diff --git a/pavelib/utils/test/suites/js_suite.py b/pavelib/utils/test/suites/js_suite.py index a6896e285854..65c5feaf843b 100644 --- a/pavelib/utils/test/suites/js_suite.py +++ b/pavelib/utils/test/suites/js_suite.py @@ -39,8 +39,6 @@ def __enter__(self): if self.mode == 'run' and not self.run_under_coverage: test_utils.clean_dir(self.report_dir) - assets.process_npm_assets() - @property def _default_subsuites(self): """ diff --git a/requirements/constraints.txt b/requirements/constraints.txt index c96ddb4ba00a..f82824d5ec6f 100644 --- a/requirements/constraints.txt +++ b/requirements/constraints.txt @@ -110,6 +110,11 @@ drf-yasg<1.21.6 # Adding pin to avoid any major upgrade djangorestframework<3.15.0 - # tests failing with greater version. Fix this in separate ticket. pillow<10.0.0 + +# Our legacy Sass code is incompatible with anything except this ancient libsass version. +# Here is a ticket to upgrade, but it's of debatable importance given that we are rapidly moving +# away from legacy LMS/CMS frontends: +# https://github.com/openedx/edx-platform/issues/31616 +libsass==0.10.0 diff --git a/requirements/edx/assets.in b/requirements/edx/assets.in new file mode 100644 index 000000000000..84dcb11611f5 --- /dev/null +++ b/requirements/edx/assets.in @@ -0,0 +1,5 @@ +-c constraints.txt + +click +libsass +nodeenv diff --git a/requirements/edx/assets.txt b/requirements/edx/assets.txt new file mode 100644 index 000000000000..4693a51a3eda --- /dev/null +++ b/requirements/edx/assets.txt @@ -0,0 +1,3 @@ +click==8.1.5 +libsass==0.10.0 +nodeenv==1.8.0 diff --git a/requirements/edx/base.txt b/requirements/edx/base.txt index 7201a2b19d9f..30f4d5f79288 100644 --- a/requirements/edx/base.txt +++ b/requirements/edx/base.txt @@ -676,10 +676,6 @@ lazy==1.5 # xblock learner-pathway-progress==1.3.4 # via -r requirements/edx/bundled.in -libsass==0.10.0 - # via - # -r requirements/edx/paver.txt - # ora2 loremipsum==1.0.5 # via ora2 lti-consumer-xblock==9.5.5 @@ -747,8 +743,6 @@ newrelic==8.8.1 # edx-django-utils nltk==3.8.1 # via chem -nodeenv==1.8.0 - # via -r requirements/edx/kernel.in numpy==1.22.4 # via # chem diff --git a/requirements/edx/development.in b/requirements/edx/development.in index ffdc299eddd9..306f3d77df4a 100644 --- a/requirements/edx/development.in +++ b/requirements/edx/development.in @@ -13,6 +13,7 @@ -r ../pip-tools.txt # pip-tools and its dependencies, for managing requirements files -r testing.txt # Dependencies for running the various test suites -r doc.txt # Dependencies for building the documentation locally. +-r assets.txt # Dependencies for rebuilding static assets. click # Used for perf_tests utilities in modulestore django-debug-toolbar # A set of panels that display debug information about the current request/response diff --git a/requirements/edx/doc.txt b/requirements/edx/doc.txt index d9aa9770b8dc..ba46533b3a47 100644 --- a/requirements/edx/doc.txt +++ b/requirements/edx/doc.txt @@ -802,10 +802,6 @@ lazy==1.5 # xblock learner-pathway-progress==1.3.4 # via -r requirements/edx/base.txt -libsass==0.10.0 - # via - # -r requirements/edx/base.txt - # ora2 loremipsum==1.0.5 # via # -r requirements/edx/base.txt @@ -887,8 +883,6 @@ nltk==3.8.1 # via # -r requirements/edx/base.txt # chem -nodeenv==1.8.0 - # via -r requirements/edx/base.txt numpy==1.22.4 # via # -r requirements/edx/base.txt diff --git a/requirements/edx/kernel.in b/requirements/edx/kernel.in index 43395d9f7efe..d6e12cab3add 100644 --- a/requirements/edx/kernel.in +++ b/requirements/edx/kernel.in @@ -108,7 +108,6 @@ mako # Primary template language used for server- Markdown # Convert text markup to HTML; used in capa problems, forums, and course wikis mongoengine # Object-document mapper for MongoDB, used in the LMS dashboard mysqlclient # Driver for the default production relational database -nodeenv # Utility for managing Node.js environments; we use this for deployments and testing oauthlib # OAuth specification support for authenticating via LTI or other Open edX services olxcleaner openedx-calc # Library supporting mathematical calculations for Open edX diff --git a/requirements/edx/paver.in b/requirements/edx/paver.in index caaf349592d2..d0c4b7dadda8 100644 --- a/requirements/edx/paver.in +++ b/requirements/edx/paver.in @@ -12,7 +12,6 @@ edx-opaque-keys # Create and introspect course and xblock identities lazy # Lazily-evaluated attributes for Python objects -libsass==0.10.0 # Python bindings for the LibSass CSS compiler markupsafe # XML/HTML/XHTML Markup safe strings mock # Stub out code with mock objects and make assertions about how they have been used path # Easier manipulation of filesystem paths diff --git a/requirements/edx/paver.txt b/requirements/edx/paver.txt index 34b6b91243b2..c75a8993bb72 100644 --- a/requirements/edx/paver.txt +++ b/requirements/edx/paver.txt @@ -16,8 +16,6 @@ idna==3.4 # via requests lazy==1.5 # via -r requirements/edx/paver.in -libsass==0.10.0 - # via -r requirements/edx/paver.in markupsafe==2.1.3 # via -r requirements/edx/paver.in mock==5.1.0 @@ -43,7 +41,6 @@ requests==2.31.0 # via -r requirements/edx/paver.in six==1.16.0 # via - # libsass # paver # python-memcached stevedore==5.1.0 diff --git a/requirements/edx/testing.txt b/requirements/edx/testing.txt index 6dab6bac6bc0..551914b9405d 100644 --- a/requirements/edx/testing.txt +++ b/requirements/edx/testing.txt @@ -869,10 +869,6 @@ lazy-object-proxy==1.9.0 # via astroid learner-pathway-progress==1.3.4 # via -r requirements/edx/base.txt -libsass==0.10.0 - # via - # -r requirements/edx/base.txt - # ora2 loremipsum==1.0.5 # via # -r requirements/edx/base.txt diff --git a/scripts/compile_sass b/scripts/compile_sass new file mode 100755 index 000000000000..d78b5a66606b --- /dev/null +++ b/scripts/compile_sass @@ -0,0 +1,413 @@ +#!/usr/bin/env python +""" +Defines a CLI for compiling Sass (both default and themed) into CSS. + +Should be run from the root of edx-platform. +Requirements for this scripts are stored in requirements/edx/compile-sass.in. + +Usage (from a provisioned environment like Tutor or Devstack): + + # Just run the script, referring to --help for details. + scripts/compile_sass --help + +Usage (standalone): + + # Active a venv if you haven't already. + python -m venv venv + . venv/bin/activate + + # Install prereqs. + pip install -r requirements/edx/compile-sass.txt + + # Run the script (refer to --help for details). + scripts/compile_sass --help + +This script is intentionally implemented in a very simplistic way. It prefers repetition +over abstraction, and its dependencies are minimal (just click and libsass-python, ideally). +We do this because: + +* If and when we migrate from libsass-python to something less ancient like node-sass or dart-sass, + we will want to re-write this script in Bash or JavaScript so that it can work without any + backend tooling. By keeping the script dead simple, that will be easier. This is the same reason + why we did not append '.py' to the name of this script. +* The features this script supports (legacy frontends & comprehensive theming) are on the way out, + in favor of micro-frontends, branding, and Paragon design tokens. We're not sure how XBlock + view styling will fit into that, but it probably can be much simpler than comprehensive theming. + So, we don't need this script to be modular and extensible. We just need it to be obvious, robust, + and easy to maintain until we can delete it. + +Why no .py extension? + +See docs/decisions/0017-reimplement-asset-processing.rst for more details. +""" +from __future__ import annotations + +import glob +import subprocess +from pathlib import Path + +import click + + +@click.option( + "-T", + "--theme-dir", + "theme_dirs", + metavar="PATH", + multiple=True, + envvar="EDX_PLATFORM_THEME_DIRS", + type=click.Path( + exists=True, file_okay=False, readable=True, writable=True, path_type=Path + ), + help=( + "Consider sub-dirs of PATH as themes. " + "Multiple theme dirs are accepted. " + "If none are provided, we look at colon-separated paths on the EDX_PLATFORM_THEME_DIRS env var." + ), +) +@click.option( + "-t", + "--theme", + "themes", + metavar="NAME", + multiple=True, + type=str, + help=( + "A theme to compile. " + "NAME should be a sub-dir of a PATH provided by --theme-dir. " + "Multiple themes are accepted. " + "If none are provided, all available themes are compiled." + ), +) +@click.option( + "--skip-default", + is_flag=True, + help="Don't compile default Sass.", +) +@click.option( + "--skip-themes", + is_flag=True, + help="Don't compile any themes (overrides --theme* options).", +) +@click.option( + "--skip-lms", + is_flag=True, + help="Don't compile any LMS Sass.", +) +@click.option( + "--skip-cms", + is_flag=True, + help="Don't compile any CMS Sass.", +) +@click.option( + "--env", + type=click.Choice(["dev", "prod"]), + default="prod", + help="Optimize CSS for this environment. Defaults to 'prod'.", +) +@click.option( + "--dry", + is_flag=True, + help="Print what would be compiled, but don't compile it.", +) +@click.option( + "-h", + "--help", + "show_help", + is_flag=True, + help="Print this help.", +) +@click.command() +@click.pass_context +def main( + context: click.Context, + theme_dirs: list[Path], + themes: list[str], + skip_default: bool, + skip_themes: bool, + skip_lms: bool, + skip_cms: bool, + env: str, + dry: bool, + show_help: bool, +) -> None: + """ + Compile Sass for edx-platform and its themes. + + Default Sass is compiled unless explicitly skipped. + Additionally, any number of themes may be specified using --theme-dir and --theme. + + Default CSS is compiled to css/ directories in edx-platform. + Themed CSS is compiled to css/ directories in their source themes. + """ + + def compile_sass_dir( + message: str, + source: Path, + dest: Path, + includes: list[Path], + tolerate_missing: bool = False, + ) -> None: + """ + TODO + """ + use_dev_settings = env == "development" + click.echo(f" {message}...") + click.echo(f" Source: {source}") + click.echo(f" Destination: {dest}") + if not source.is_dir(): + if tolerate_missing: + click.echo(f" Skipped because source directory does not exist.") + return + else: + raise FileNotFoundError(f"missing Sass source dir: {source}") + click.echo(f" Include paths:") + for include in includes: + click.echo(f" {include}") + if not dry: + # Import sass late so that this script can be dry-run without installing + # libsass, which takes a while as it must be compiled from its C source. + import sass + + dest.mkdir(parents=True, exist_ok=True) + sass.compile( + dirname=(str(source), str(dest)), + include_paths=[str(include_path) for include_path in includes], + source_comments=use_dev_settings, + output_style=("nested" if use_dev_settings else "compressed"), + ) + click.echo(f" Compiled.") + # For Sass files without explicit RTL versions, generate + # an RTL version of the CSS using the rtlcss library. + for sass_path in glob.glob(str(source) + "/**/*.scss"): + if Path(sass_path).name.startswith("_"): + # Don't generate RTL CSS for partials + continue + if sass_path.endswith("-rtl.scss"): + # Don't generate RTL CSS if the file is itself an RTL version + continue + if Path(sass_path.replace(".scss", "-rtl.scss")).exists(): + # Don't generate RTL CSS if there is an explicit Sass version for RTL + continue + click.echo(" Generating missing right-to-left Sass:") + source_css_file = sass_path.replace(str(source), str(dest)).replace( + ".scss", ".css" + ) + target_css_file = source_css_file.replace(".css", "-rtl.css") + click.echo(f" {source_css_file} -> ") + click.echo(f" {target_css_file}") + if not dry: + subprocess.run(["rtlcss", source_css_file, target_css_file]) + + if show_help: + click.echo(context.get_help()) + return + if skip_lms and skip_cms: + click.echo("Skipping both LMS and CMS... nothing to do! Will exit.") + return + + # Build a list of theme paths: + if skip_themes: + theme_paths = [] + else: + theme_paths = [ + theme_dir / theme + # For every theme dir, + for theme_dir in theme_dirs + for theme in ( + # for every theme name (if theme names provided), + themes + or + # or for every subdir of theme dirs (if no theme name provided), + (theme_dir_entry.name for theme_dir_entry in theme_dir.iterdir()) + ) + # consider the path a theme if it has a lms/ or cms/ subdirectory. + if (theme_dir / theme / "lms").is_dir() or (theme_dir / theme / "cms").is_dir() + ] + + # We expect this script to be run from the edx-platform root. + repo = Path(".") + if not (repo / "xmodule").is_dir(): + # Sanity check: If the xmodule/ folder is missing, we're definitely not at the root + # of edx-platform, so save the user some headache by exiting early. + raise Exception(f"{__file__} must be run from the root of edx-platform") + + # Every Sass compilation will use have these directories as lookup paths, + # regardless of theme. + common_includes = [ + repo / "common" / "static", + repo / "common" / "static" / "sass", + repo / "node_modules" / "@edx", + repo / "node_modules", + ] + + if not skip_default: + click.echo(f"Compiling default Sass...") + if not skip_lms: + compile_sass_dir( + "Compiling default LMS Sass", + repo / "lms" / "static" / "sass", + repo / "lms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + ], + ) + compile_sass_dir( + "Compiling default certificate Sass", + repo / "lms" / "static" / "certificates" / "sass", + repo / "lms" / "static" / "certificates" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + ], + ) + compile_sass_dir( + "Compiling built-in XBlock Sass for default LMS", + repo / "xmodule" / "assets", + repo / "lms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + repo / "cms" / "static" / "sass", + ], + ) + if not skip_cms: + compile_sass_dir( + "Compiling default CMS Sass", + repo / "cms" / "static" / "sass", + repo / "cms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass", + ], + ) + compile_sass_dir( + "Compiling built-in XBlock Sass for default CMS", + repo / "xmodule" / "assets", + repo / "cms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + repo / "cms" / "static" / "sass", + ], + ) + click.echo(f"Done compiling default Sass!") + + for theme in theme_paths: + click.echo(f"Compiling Sass for theme at {theme}...") + if not skip_lms: + compile_sass_dir( + "Compiling default LMS Sass with themed partials", + repo / "lms" / "static" / "sass", + theme / "lms" / "static" / "css", + includes=[ + *common_includes, + theme / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + ], + tolerate_missing=True, + ) + compile_sass_dir( + "Compiling themed LMS Sass as overrides to CSS from previous step", + theme / "lms" / "static" / "sass", + theme / "lms" / "static" / "css", + includes=[ + *common_includes, + theme / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + ], + tolerate_missing=True, + ) + compile_sass_dir( + "Compiling themed certificate Sass", + theme / "lms" / "static" / "certificates" / "sass", + theme / "lms" / "static" / "certificates" / "css", + includes=[ + *common_includes, + theme / "lms" / "static" / "sass" / "partials", + theme / "lms" / "static" / "sass", + ], + tolerate_missing=True, + ) + compile_sass_dir( + "Compiling built-in XBlock Sass for themed LMS", + repo / "xmodule" / "assets", + theme / "lms" / "static" / "css", + includes=[ + *common_includes, + theme / "lms" / "static" / "sass" / "partials", + theme / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + repo / "cms" / "static" / "sass", + ], + ) + if not skip_cms: + compile_sass_dir( + "Compiling default CMS Sass with themed partials", + repo / "cms" / "static" / "sass", + theme / "cms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + theme / "cms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass", + ], + tolerate_missing=True, + ) + compile_sass_dir( + "Compiling themed CMS Sass as overrides to CSS from previous step", + theme / "cms" / "static" / "sass", + theme / "cms" / "static" / "css", + includes=[ + *common_includes, + repo / "lms" / "static" / "sass" / "partials", + theme / "cms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass", + ], + tolerate_missing=True, + ) + compile_sass_dir( + "Compiling built-in XBlock Sass for themed CMS", + repo / "xmodule" / "assets", + theme / "cms" / "static" / "css", + includes=[ + *common_includes, + theme / "lms" / "static" / "sass" / "partials", + theme / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass" / "partials", + repo / "cms" / "static" / "sass" / "partials", + repo / "lms" / "static" / "sass", + repo / "cms" / "static" / "sass", + ], + ) + click.echo(f"Done compiling Sass for theme at {theme}!") + + click.echo() + click.echo("Successfully compiled:") + if not skip_default: + click.echo(f" - {repo.absolute()} (default Sass)") + for theme in theme_paths: + click.echo(f" - {theme}") + if skip_lms: + click.echo(f"(skipped LMS)") + if skip_cms: + click.echo(f"(skipped CMS)") + + +if __name__ == "__main__": + main() diff --git a/scripts/copy-node-modules.sh b/scripts/copy-node-modules.sh new file mode 100755 index 000000000000..db997da95785 --- /dev/null +++ b/scripts/copy-node-modules.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copy certain npm-installed assets from node_modules to other folders in +# edx-platform. These assets are used by certain especially-old legacy LMS & CMS +# frontends that are not set up to import from node_modules directly. +# Many of the destination folders are named "vendor", because they originally +# held vendored-in (directly-committed) libraries; once we moved most frontends +# to use NPM, we decided to keep library versions in-sync by copying to the +# former "vendor" directories. + +# Enable stricter error handling. +set -euo pipefail + +COL_LOG="\e[36m" # Log/step/section color (cyan) +COL_OFF="\e[0m" # Normal color + +# Keep these as variables in case we ever want to parameterize this script's +# input or output dirs, as proposed in: +# https://github.com/openedx/wg-developer-experience/issues/150 +# https://github.com/openedx/wg-developer-experience/issues/151 +node_modules="node_modules" +vendor_js="common/static/common/js/vendor" +vendor_css="common/static/common/css/vendor" + +# Stylized logging. +log ( ) { + echo -e "${COL_LOG}$* $COL_OFF" +} + +log "=====================================================================================" +log "Copying required assets from node_modules..." +log "-------------------------------------------------------------------------------" + +# Start echoing all commands back to user for ease of debugging. +set -x + +log "Ensuring vendor directories exist..." +mkdir -p "$vendor_js" +mkdir -p "$vendor_css" + +log "Copying studio-frontend JS & CSS from node_modules into vendor directores..." +while read -r -d $'\0' src_file ; do + if [[ "$src_file" = *.css ]] || [[ "$src_file" = *.css.map ]] ; then + cp --force "$src_file" "$vendor_css" + else + cp --force "$src_file" "$vendor_js" + fi +done < <(find "$node_modules/@edx/studio-frontend/dist" -type f -print0) + +log "Copying certain JS modules from node_modules into vendor directory..." +cp --force \ + "$node_modules/backbone.paginator/lib/backbone.paginator.js" \ + "$node_modules/backbone/backbone.js" \ + "$node_modules/bootstrap/dist/js/bootstrap.bundle.js" \ + "$node_modules/hls.js/dist/hls.js" \ + "$node_modules/jquery-migrate/dist/jquery-migrate.js" \ + "$node_modules/jquery.scrollto/jquery.scrollTo.js" \ + "$node_modules/jquery/dist/jquery.js" \ + "$node_modules/moment-timezone/builds/moment-timezone-with-data.js" \ + "$node_modules/moment/min/moment-with-locales.js" \ + "$node_modules/picturefill/dist/picturefill.js" \ + "$node_modules/requirejs/require.js" \ + "$node_modules/underscore.string/dist/underscore.string.js" \ + "$node_modules/underscore/underscore.js" \ + "$node_modules/which-country/index.js" \ + "$vendor_js" + +log "Copying certain JS developer modules into vendor directory..." +if [[ "${NODE_ENV:-production}" = development ]] ; then + cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js" + cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js" +else + # TODO: https://github.com/openedx/edx-platform/issues/31768 + # In the old implementation of this scipt (pavelib/assets.py), these two + # developer libraries were copied into the JS vendor directory whether not + # the build was for prod or dev. In order to exactly match the output of + # the old script, this script will also copy them in for prod builds. + # However, in the future, it would be good to only copy them for dev + # builds. Furthermore, these libraries should not be `npm install`ed + # into prod builds in the first place. + cp --force "$node_modules/sinon/pkg/sinon.js" "$vendor_js" || true # "|| true" means "tolerate errors"; in this case, + cp --force "$node_modules/squirejs/src/Squire.js" "$vendor_js" || true # that's "tolerate if these files don't exist." +fi + +# Done echoing. +set +x + +log "-------------------------------------------------------------------------------" +log " Done copying required assets from node_modules." +log "=====================================================================================" + diff --git a/setup.py b/setup.py index 17d0f07b01e1..119509da1374 100644 --- a/setup.py +++ b/setup.py @@ -186,8 +186,5 @@ ], 'xblock.v1': XBLOCKS, 'xblock_asides.v1': XBLOCKS_ASIDES, - 'console_scripts': [ - 'xmodule_assets = xmodule.static_content:main', - ], } ) diff --git a/webpack.common.config.js b/webpack.common.config.js index bebd07cbb599..01e033e73c5a 100644 --- a/webpack.common.config.js +++ b/webpack.common.config.js @@ -9,7 +9,7 @@ var StringReplace = require('string-replace-webpack-plugin'); var Merge = require('webpack-merge'); var files = require('./webpack-config/file-lists.js'); -var xmoduleJS = require('./common/static/xmodule/webpack.xmodule.config.js'); +var xmoduleJS = require('./webpack.xmodule.config.js'); var filesWithRequireJSBlocks = [ path.resolve(__dirname, 'common/static/common/js/components/utils/view_utils.js'), @@ -26,6 +26,9 @@ var defineFooter = new RegExp('(' + defineCallFooter.source + ')|(' + defineDirectFooter.source + ')|(' + defineFancyFooter.source + ')', 'm'); +var staticRootLms = process.env.STATIC_ROOT_LMS || "./test_root/staticfiles"; +var staticRootCms = process.env.STATIC_ROOT_CMS || (staticRootLms + "/studio"); + var workerConfig = function() { try { return { @@ -39,7 +42,7 @@ var workerConfig = function() { }, plugins: [ new BundleTracker({ - path: process.env.STATIC_ROOT_LMS, + path: staticRootLms, filename: 'webpack-worker-stats.json' }), new webpack.DefinePlugin({ @@ -131,14 +134,15 @@ module.exports = Merge.smart({ }, plugins: [ + new webpack.ProgressPlugin(), // report progress during compilation new webpack.NoEmitOnErrorsPlugin(), new webpack.NamedModulesPlugin(), new BundleTracker({ - path: process.env.STATIC_ROOT_CMS, + path: staticRootCms, filename: 'webpack-stats.json' }), new BundleTracker({ - path: process.env.STATIC_ROOT_LMS, + path: staticRootLms, filename: 'webpack-stats.json' }), new webpack.ProvidePlugin({ diff --git a/webpack.dev.config.js b/webpack.dev.config.js index 3987e82fdf7d..90018a901002 100644 --- a/webpack.dev.config.js +++ b/webpack.dev.config.js @@ -21,7 +21,7 @@ module.exports = _.values(Merge.smart(commonConfig, { }), new webpack.DefinePlugin({ 'process.env.NODE_ENV': JSON.stringify('development'), - 'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG + 'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG || "{}" }) ], module: { diff --git a/webpack.prod.config.js b/webpack.prod.config.js index d71031cffc43..cff1f23699fd 100644 --- a/webpack.prod.config.js +++ b/webpack.prod.config.js @@ -18,7 +18,7 @@ var optimizedConfig = Merge.smart(commonConfig, { plugins: [ new webpack.DefinePlugin({ 'process.env.NODE_ENV': JSON.stringify('production'), - 'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG + 'process.env.JS_ENV_EXTRA_CONFIG': process.env.JS_ENV_EXTRA_CONFIG || "{}" }), new webpack.LoaderOptionsPlugin({ // This may not be needed; legacy option for loaders written for webpack 1 minimize: true diff --git a/webpack.xmodule.config.js b/webpack.xmodule.config.js new file mode 100644 index 000000000000..30f5bdb2a507 --- /dev/null +++ b/webpack.xmodule.config.js @@ -0,0 +1,136 @@ +module.exports = { + "entry": { + "AboutBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js", + "./xmodule/js/src/html/imageModal.js", + "./xmodule/js/common_static/js/vendor/draggabilly.js" + ], + "AboutBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/edit.js" + ], + "AnnotatableBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/display.js", + "./xmodule/js/src/annotatable/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js" + ], + "AnnotatableBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/raw/edit/xml.js" + ], + "ConditionalBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/conditional/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js" + ], + "ConditionalBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/sequence/edit.js" + ], + "CourseInfoBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js", + "./xmodule/js/src/html/imageModal.js", + "./xmodule/js/common_static/js/vendor/draggabilly.js" + ], + "CourseInfoBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/edit.js" + ], + "CustomTagBlockDisplay": "./xmodule/js/src/xmodule.js", + "CustomTagBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/raw/edit/xml.js" + ], + "HtmlBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js", + "./xmodule/js/src/html/imageModal.js", + "./xmodule/js/common_static/js/vendor/draggabilly.js" + ], + "HtmlBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/edit.js" + ], + "LTIBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/lti/lti.js" + ], + "LTIBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/raw/edit/metadata-only.js" + ], + "LibraryContentBlockDisplay": "./xmodule/js/src/xmodule.js", + "LibraryContentBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/vertical/edit.js" + ], + "PollBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/poll/poll.js", + "./xmodule/js/src/poll/poll_main.js" + ], + "PollBlockEditor": "./xmodule/js/src/xmodule.js", + "ProblemBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/capa/display.js", + "./xmodule/js/src/collapsible.js", + "./xmodule/js/src/capa/imageinput.js", + "./xmodule/js/src/capa/schematic.js" + ], + "ProblemBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/problem/edit.js" + ], + "SequenceBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/sequence/display.js" + ], + "SequenceBlockEditor": "./xmodule/js/src/xmodule.js", + "SplitTestBlockDisplay": "./xmodule/js/src/xmodule.js", + "SplitTestBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/sequence/edit.js" + ], + "StaticTabBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/display.js", + "./xmodule/js/src/javascript_loader.js", + "./xmodule/js/src/collapsible.js", + "./xmodule/js/src/html/imageModal.js", + "./xmodule/js/common_static/js/vendor/draggabilly.js" + ], + "StaticTabBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/html/edit.js" + ], + "VideoBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/video/10_main.js" + ], + "VideoBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/tabs/tabs-aggregator.js" + ], + "WordCloudBlockDisplay": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/assets/word_cloud/src/js/word_cloud.js" + ], + "WordCloudBlockEditor": [ + "./xmodule/js/src/xmodule.js", + "./xmodule/js/src/raw/edit/metadata-only.js" + ] + } +}; diff --git a/xmodule/annotatable_block.py b/xmodule/annotatable_block.py index 3c78a3b7c0e7..95f68d4370b6 100644 --- a/xmodule/annotatable_block.py +++ b/xmodule/annotatable_block.py @@ -4,7 +4,6 @@ import textwrap from lxml import etree -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from xblock.core import XBlock from xblock.fields import Scope, String @@ -15,7 +14,6 @@ from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -35,7 +33,6 @@ class AnnotatableBlock( XmlMixin, EditingMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): @@ -73,22 +70,6 @@ class AnnotatableBlock( uses_xmodule_styles_setup = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/html/display.js'), - resource_filename(__name__, 'js/src/annotatable/display.js'), - resource_filename(__name__, 'js/src/javascript_loader.js'), - resource_filename(__name__, 'js/src/collapsible.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/raw/edit/xml.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } studio_js_module_name = "XMLEditingDescriptor" mako_template = "widgets/raw-edit.html" diff --git a/xmodule/assets/README.rst b/xmodule/assets/README.rst index 618ba029246f..806010eaf73b 100644 --- a/xmodule/assets/README.rst +++ b/xmodule/assets/README.rst @@ -41,8 +41,8 @@ the corresponding folders in any enabled themes, as part of the edx-platform bui It is collected into the static root, and then linked to from XBlock fragments by the ``add_sass_to_fragment`` function in `builtin_assets.py`_. -.. _AnnotatableBlockDisplay: https://github.com/openedx/edx-platform/tree/master/xmodule/assets/AnnotatableBlockDisplay.scss -.. _AnnotatableBlockEditor: https://github.com/openedx/edx-platform/tree/master/xmodule/assets/AnnotatableBlockEditor.scss +.. _AnnotatableBlockDisplay.scss: https://github.com/openedx/edx-platform/tree/master/xmodule/assets/AnnotatableBlockDisplay.scss +.. _AnnotatableBlockEditor.scss: https://github.com/openedx/edx-platform/tree/master/xmodule/assets/AnnotatableBlockEditor.scss .. _annotatable/_display.scss: https://github.com/openedx/edx-platform/tree/master/xmodule/assets/annotatable/_display.scss .. _simplify things: https://github.com/openedx/edx-platform/issues/32621 @@ -59,8 +59,7 @@ Currently, edx-platform XBlock JS is defined both here in `xmodule/assets`_ and * For many older blocks, their JS is: - * copied to ``common/static/xmodule`` by `static_content.py`_ (aka ``xmodule_assets``), - * then bundled using a generated Webpack config at ``common/static/xmodule/webpack.xmodule.config.js``, + * bundled using a `webpack.xmodule.config.js`_, * which is included into `webpack.common.config.js`_, * allowing it to be included into XBlock fragments using ``add_webpack_js_to_fragment`` from `builtin_assets.py`_. @@ -75,12 +74,7 @@ Currently, edx-platform XBlock JS is defined both here in `xmodule/assets`_ and * `VerticalBlock`_ * `LibrarySourcedBlock`_ -As part of an `active build refactoring`_: - -* We update the older builtin XBlocks to reference their JS directly rather than using copies of it. -* We will move ``webpack.xmodule.config.js`` here instead of generating it. -* We will consolidate all edx-platform XBlock JS here in `xmodule/assets`_. -* We will delete the ``xmodule_assets`` script. +As part of an `active build refactoring`_, we will soon consolidate all edx-platform XBlock JS here in `xmodule/assets`_. .. _xmodule/assets: https://github.com/openedx/edx-platform/tree/master/xmodule/assets .. _xmodule/js: https://github.com/openedx/edx-platform/tree/master/xmodule/js @@ -93,4 +87,5 @@ As part of an `active build refactoring`_: .. _builtin_assets.py: https://github.com/openedx/edx-platform/tree/master/xmodule/util/builtin_assets.py .. _static_content.py: https://github.com/openedx/edx-platform/blob/master/xmodule/static_content.py .. _library_source_block/style.css: https://github.com/openedx/edx-platform/blob/master/xmodule/assets/library_source_block/style.css +.. _webpack.xmodule.config.js: https://github.com/openedx/edx-platform/blob/master/webpack.xmodule.config.js .. _webpack.common.config.js: https://github.com/openedx/edx-platform/blob/master/webpack.common.config.js diff --git a/xmodule/capa_block.py b/xmodule/capa_block.py index 43b5bb3efa42..9b69556e505d 100644 --- a/xmodule/capa_block.py +++ b/xmodule/capa_block.py @@ -19,7 +19,6 @@ from django.utils.encoding import smart_str from django.utils.functional import cached_property from lxml import etree -from pkg_resources import resource_filename from pytz import utc from web_fragments.fragment import Fragment from xblock.core import XBlock @@ -39,7 +38,6 @@ from xmodule.util.sandboxing import SandboxService from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, XModuleMixin, XModuleToXBlockMixin, @@ -131,7 +129,6 @@ class ProblemBlock( XmlMixin, EditingMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): @@ -165,24 +162,6 @@ class ProblemBlock( uses_xmodule_styles_setup = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/javascript_loader.js'), - resource_filename(__name__, 'js/src/capa/display.js'), - resource_filename(__name__, 'js/src/collapsible.js'), - resource_filename(__name__, 'js/src/capa/imageinput.js'), - resource_filename(__name__, 'js/src/capa/schematic.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js') - } - - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/problem/edit.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - display_name = String( display_name=_("Display Name"), help=_("The display name for this component."), diff --git a/xmodule/conditional_block.py b/xmodule/conditional_block.py index 5f484050d438..03662deb01e9 100644 --- a/xmodule/conditional_block.py +++ b/xmodule/conditional_block.py @@ -9,7 +9,6 @@ from lazy import lazy from lxml import etree from opaque_keys.edx.locator import BlockUsageLocator -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from xblock.core import XBlock from xblock.fields import ReferenceList, Scope, String @@ -23,7 +22,6 @@ from xmodule.validation import StudioValidation, StudioValidationMessage from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, STUDENT_VIEW, @@ -44,7 +42,6 @@ class ConditionalBlock( MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, StudioEditableBlock, @@ -146,21 +143,8 @@ class ConditionalBlock( show_in_read_only_mode = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/conditional/display.js'), - resource_filename(__name__, 'js/src/javascript_loader.js'), - resource_filename(__name__, 'js/src/collapsible.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - mako_template = 'widgets/metadata-edit.html' studio_js_module_name = 'SequenceDescriptor' - studio_view_js = { - 'js': [resource_filename(__name__, 'js/src/sequence/edit.js')], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } # Map # key: diff --git a/xmodule/discussion_block.py b/xmodule/discussion_block.py index 84d5fdfb4afb..fd9dfea7d6b5 100644 --- a/xmodule/discussion_block.py +++ b/xmodule/discussion_block.py @@ -15,11 +15,13 @@ from xblockutils.resources import ResourceLoader from xblockutils.studio_editable import StudioEditableXBlockMixin +from lms.djangoapps.discussion.django_comment_client.permissions import has_permission from openedx.core.djangoapps.discussions.models import DiscussionsConfiguration, Provider from openedx.core.djangolib.markup import HTML, Text from openedx.core.lib.xblock_utils import get_css_dependencies, get_js_dependencies from xmodule.xml_block import XmlMixin + log = logging.getLogger(__name__) loader = ResourceLoader(__name__) # pylint: disable=invalid-name @@ -154,9 +156,6 @@ def has_permission(self, permission): :param str permission: Permission :rtype: bool """ - # normal import causes the xmodule_assets command to fail due to circular import - hence importing locally - from lms.djangoapps.discussion.django_comment_client.permissions import has_permission - return has_permission(self.django_user, permission, self.course_key) def student_view(self, context=None): diff --git a/xmodule/error_block.py b/xmodule/error_block.py index fcebf8f744ff..a3379007e19f 100644 --- a/xmodule/error_block.py +++ b/xmodule/error_block.py @@ -18,7 +18,6 @@ from xmodule.errortracker import exc_info_to_str from xmodule.modulestore import EdxJSONEncoder from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, XModuleMixin, XModuleToXBlockMixin, @@ -47,7 +46,6 @@ class ErrorFields: class ErrorBlock( ErrorFields, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): # pylint: disable=abstract-method diff --git a/xmodule/html_block.py b/xmodule/html_block.py index 3027c24b2ba8..688e515c80e9 100644 --- a/xmodule/html_block.py +++ b/xmodule/html_block.py @@ -8,8 +8,6 @@ import textwrap from datetime import datetime -from pkg_resources import resource_filename - from django.conf import settings from fs.errors import ResourceNotFound from lxml import etree @@ -27,7 +25,6 @@ from xmodule.util.misc import escape_html_characters from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -47,7 +44,7 @@ @XBlock.needs("user") class HtmlBlockMixin( # lint-amnesty, pylint: disable=abstract-method XmlMixin, EditingMixin, - XModuleToXBlockMixin, HTMLSnippet, ResourceTemplates, XModuleMixin, + XModuleToXBlockMixin, ResourceTemplates, XModuleMixin, ): """ The HTML XBlock mixin. @@ -144,17 +141,6 @@ def studio_view(self, _context): shim_xmodule_js(fragment, 'HTMLEditingDescriptor') return fragment - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/html/display.js'), - resource_filename(__name__, 'js/src/javascript_loader.js'), - resource_filename(__name__, 'js/src/collapsible.js'), - resource_filename(__name__, 'js/src/html/imageModal.js'), - resource_filename(__name__, 'js/common_static/js/vendor/draggabilly.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - uses_xmodule_styles_setup = True mako_template = "widgets/html-edit.html" @@ -163,13 +149,6 @@ def studio_view(self, _context): template_dir_name = "html" show_in_read_only_mode = True - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/html/edit.js') - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - # VS[compat] TODO (cpennington): Delete this method once all fall 2012 course # are being edited in the cms @classmethod diff --git a/xmodule/library_content_block.py b/xmodule/library_content_block.py index 74ab77630626..047d36abca52 100644 --- a/xmodule/library_content_block.py +++ b/xmodule/library_content_block.py @@ -17,7 +17,6 @@ from lxml import etree from lxml.etree import XMLSyntaxError from opaque_keys.edx.locator import LibraryLocator -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from webob import Response from xblock.completable import XBlockCompletionMode @@ -31,7 +30,6 @@ from xmodule.validation import StudioValidation, StudioValidationMessage from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, STUDENT_VIEW, @@ -76,7 +74,6 @@ class LibraryContentBlock( MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, StudioEditableBlock, @@ -95,19 +92,8 @@ class LibraryContentBlock( resources_dir = 'assets/library_content' - preview_view_js = { - 'js': [], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - mako_template = 'widgets/metadata-edit.html' studio_js_module_name = "VerticalDescriptor" - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/vertical/edit.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } show_in_read_only_mode = True diff --git a/xmodule/lti_block.py b/xmodule/lti_block.py index 80afed7a0887..4228aec74393 100644 --- a/xmodule/lti_block.py +++ b/xmodule/lti_block.py @@ -68,7 +68,6 @@ from django.conf import settings from lxml import etree from oauthlib.oauth1.rfc5849 import signature -from pkg_resources import resource_filename from pytz import UTC from webob import Response from web_fragments.fragment import Fragment @@ -88,7 +87,6 @@ from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -284,7 +282,6 @@ class LTIBlock( EditingMixin, MakoTemplateBlockBase, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): # pylint: disable=abstract-method @@ -372,22 +369,9 @@ class LTIBlock( resources_dir = None uses_xmodule_styles_setup = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/lti/lti.js') - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - mako_template = 'widgets/metadata-only-edit.html' studio_js_module_name = 'MetadataOnlyEditingDescriptor' - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/raw/edit/metadata-only.js') - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } def studio_view(self, _context): """ diff --git a/xmodule/poll_block.py b/xmodule/poll_block.py index 6093237fa3c5..c4d9372883ee 100644 --- a/xmodule/poll_block.py +++ b/xmodule/poll_block.py @@ -13,7 +13,6 @@ from collections import OrderedDict from copy import deepcopy -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from lxml import etree @@ -24,7 +23,6 @@ from xmodule.stringify import stringify_children from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -42,7 +40,6 @@ class PollBlock( MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): # pylint: disable=abstract-method @@ -84,22 +81,6 @@ class PollBlock( resources_dir = None uses_xmodule_styles_setup = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/javascript_loader.js'), - resource_filename(__name__, 'js/src/poll/poll.js'), - resource_filename(__name__, 'js/src/poll/poll_main.js') - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - - # There is no studio_view() for this XBlock but this is needed to make the - # the static_content command happy. - studio_view_js = { - 'js': [], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js') - } - def handle_ajax(self, dispatch, data): # lint-amnesty, pylint: disable=unused-argument """Ajax handler. diff --git a/xmodule/randomize_block.py b/xmodule/randomize_block.py index c5e1f2c4747f..b8a1432ff311 100644 --- a/xmodule/randomize_block.py +++ b/xmodule/randomize_block.py @@ -11,7 +11,6 @@ from xmodule.seq_block import SequenceMixin from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, STUDENT_VIEW, XModuleMixin, @@ -26,7 +25,6 @@ class RandomizeBlock( MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): diff --git a/xmodule/seq_block.py b/xmodule/seq_block.py index b0a2789fd273..5561d034da87 100644 --- a/xmodule/seq_block.py +++ b/xmodule/seq_block.py @@ -14,7 +14,6 @@ from lxml import etree from opaque_keys.edx.keys import UsageKey -from pkg_resources import resource_filename from pytz import UTC from web_fragments.fragment import Fragment from xblock.completable import XBlockCompletionMode @@ -25,7 +24,6 @@ from edx_toggles.toggles import WaffleFlag, SettingDictToggle from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, STUDENT_VIEW, @@ -258,7 +256,6 @@ class SequenceBlock( MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): @@ -271,20 +268,6 @@ class SequenceBlock( show_in_read_only_mode = True uses_xmodule_styles_setup = True - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/sequence/display.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js') - } - - # There is no studio_view() for this XBlock but this is needed to make the - # the static_content command happy. - studio_view_js = { - 'js': [], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js') - } - def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) diff --git a/xmodule/split_test_block.py b/xmodule/split_test_block.py index 5984ae531efc..8984dc62dbb5 100644 --- a/xmodule/split_test_block.py +++ b/xmodule/split_test_block.py @@ -12,7 +12,6 @@ from django.utils.functional import cached_property from lxml import etree -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from webob import Response from xblock.core import XBlock @@ -26,7 +25,6 @@ from xmodule.validation import StudioValidation, StudioValidationMessage from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, STUDENT_VIEW, @@ -132,7 +130,6 @@ class SplitTestBlock( # lint-amnesty, pylint: disable=abstract-method MakoTemplateBlockBase, XmlMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, StudioEditableBlock, @@ -158,17 +155,8 @@ class SplitTestBlock( # lint-amnesty, pylint: disable=abstract-method show_in_read_only_mode = True - preview_view_js = { - 'js': [], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - mako_template = "widgets/metadata-only-edit.html" studio_js_module_name = 'SequenceDescriptor' - studio_view_js = { - 'js': [resource_filename(__name__, 'js/src/sequence/edit.js')], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } @cached_property def child_block(self): diff --git a/xmodule/static_content.py b/xmodule/static_content.py deleted file mode 100755 index 03014476737a..000000000000 --- a/xmodule/static_content.py +++ /dev/null @@ -1,254 +0,0 @@ -# /usr/bin/env python -""" -This module has utility functions for gathering up the javascript -that is defined by XModules and XModuleDescriptors -""" - - -import errno -import hashlib -import json -import logging -import os -import sys -import textwrap -from collections import defaultdict -from pkg_resources import resource_filename - -import django -from path import Path as path - -from xmodule.annotatable_block import AnnotatableBlock -from xmodule.capa_block import ProblemBlock -from xmodule.conditional_block import ConditionalBlock -from xmodule.html_block import AboutBlock, CourseInfoBlock, HtmlBlock, StaticTabBlock -from xmodule.library_content_block import LibraryContentBlock -from xmodule.lti_block import LTIBlock -from xmodule.poll_block import PollBlock -from xmodule.seq_block import SequenceBlock -from xmodule.split_test_block import SplitTestBlock -from xmodule.template_block import CustomTagBlock -from xmodule.word_cloud_block import WordCloudBlock -from xmodule.x_module import HTMLSnippet - -LOG = logging.getLogger(__name__) - - -class VideoBlock(HTMLSnippet): # lint-amnesty, pylint: disable=abstract-method - """ - Static assets for VideoBlock. - Kept here because importing VideoBlock code requires Django to be setup. - """ - - preview_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/video/10_main.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js') - } - - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/tabs/tabs-aggregator.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - - -# List of XBlocks which use this static content setup. -# Should only be used for XModules being converted to XBlocks. -XBLOCK_CLASSES = [ - AboutBlock, - AnnotatableBlock, - ConditionalBlock, - CourseInfoBlock, - CustomTagBlock, - HtmlBlock, - LibraryContentBlock, - LTIBlock, - PollBlock, - ProblemBlock, - SequenceBlock, - SplitTestBlock, - StaticTabBlock, - VideoBlock, - WordCloudBlock, -] - - -def write_module_js(output_root): - """Write all registered XModule js and coffee files to output root.""" - return _write_js(output_root, XBLOCK_CLASSES, 'get_preview_view_js') - - -def write_descriptor_js(output_root): - """Write all registered XModuleDescriptor js and coffee files to output root.""" - return _write_js(output_root, XBLOCK_CLASSES, 'get_studio_view_js') - - -def _ensure_dir(directory): - """Ensure that `directory` exists.""" - try: - os.makedirs(directory) - except OSError as exc: - if exc.errno == errno.EEXIST: - pass - else: - raise - - -def _write_js(output_root, classes, js_attribute): - """ - Write the javascript fragments from all XModules in `classes` - into `output_root` as individual files, hashed by the contents to remove - duplicates - - Returns a dictionary mapping class names to the files that they depend on. - """ - file_contents = {} - file_owners = defaultdict(list) - - fragment_owners = defaultdict(list) - for class_ in classes: - module_js = getattr(class_, js_attribute)() - with open(module_js.get('xmodule_js'), 'rb') as xmodule_js_file: - xmodule_js_fragment = xmodule_js_file.read() - # It will enforce 000 prefix for xmodule.js. - fragment_owners[(0, 'js', xmodule_js_fragment)].append(getattr(class_, js_attribute + '_bundle_name')()) - for filetype in ('coffee', 'js'): - for idx, fragment_path in enumerate(module_js.get(filetype, [])): - with open(fragment_path, 'rb') as fragment_file: - fragment = fragment_file.read() - fragment_owners[(idx + 1, filetype, fragment)].append(getattr(class_, js_attribute + '_bundle_name')()) - - for (idx, filetype, fragment), owners in sorted(fragment_owners.items()): - filename = "{idx:0=3d}-{hash}.{type}".format( - idx=idx, - hash=hashlib.md5(fragment).hexdigest(), - type=filetype) - file_contents[filename] = fragment - for owner in owners: - file_owners[owner].append(output_root / filename) - - _write_files(output_root, file_contents, {'.coffee': '.js'}) - - return file_owners - - -def _write_files(output_root, contents, generated_suffix_map=None): - """ - Write file contents to output root. - - Any files not listed in contents that exists in output_root will be deleted, - unless it matches one of the patterns in `generated_suffix_map`. - - output_root (path): The root directory to write the file contents in - contents (dict): A map from filenames to file contents to be written to the output_root - generated_suffix_map (dict): Optional. Maps file suffix to generated file suffix. - For any file in contents, if the suffix matches a key in `generated_suffix_map`, - then the same filename with the suffix replaced by the value from `generated_suffix_map` - will be ignored - """ - _ensure_dir(output_root) - to_delete = {file.basename() for file in output_root.files()} - set(contents.keys()) - - if generated_suffix_map: - for output_file in contents.keys(): - for suffix, generated_suffix in generated_suffix_map.items(): - if output_file.endswith(suffix): - to_delete.discard(output_file.replace(suffix, generated_suffix)) - - for extra_file in to_delete: - (output_root / extra_file).remove_p() - - for filename, file_content in contents.items(): - output_file = output_root / filename - - not_file = not output_file.isfile() - - # Sometimes content is already unicode and sometimes it's not - # so we add this conditional here to make sure that below we're - # always working with streams of bytes. - if not isinstance(file_content, bytes): - file_content = file_content.encode('utf-8') - - # not_file is included to short-circuit this check, because - # read_md5 depends on the file already existing - write_file = not_file or output_file.read_md5() != hashlib.md5(file_content).digest() - if write_file: - LOG.debug("Writing %s", output_file) - output_file.write_bytes(file_content) - else: - LOG.debug("%s unchanged, skipping", output_file) - - -def write_webpack(output_file, module_files, descriptor_files): - """ - Write all xmodule and xmodule descriptor javascript into module-specific bundles. - - The output format should be suitable for smart-merging into an existing webpack configuration. - """ - _ensure_dir(output_file.dirname()) - - config = { - 'entry': {} - } - for (owner, files) in list(module_files.items()) + list(descriptor_files.items()): - unique_files = sorted({f'./{file}' for file in files}) - if len(unique_files) == 1: - unique_files = unique_files[0] - config['entry'][owner] = unique_files - # config['entry']['modules/js/all'] = sorted(set('./{}'.format(file) for file in sum(module_files.values(), []))) - # config['entry']['descriptors/js/all'] = sorted(set('./{}'.format(file) for file in sum(descriptor_files.values(), []))) # lint-amnesty, pylint: disable=line-too-long - - with output_file.open('w') as outfile: - outfile.write( - textwrap.dedent("""\ - module.exports = {config_json}; - """).format( - config_json=json.dumps( - config, - indent=4, - sort_keys=True, - ) - ) - ) - - -def main(): - """ - Generate - Usage: static_content.py - """ - from django.conf import settings - # Install only the apps whose models are imported when this runs - installed_apps = ( - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'config_models', - 'openedx.core.djangoapps.video_config', - 'openedx.core.djangoapps.video_pipeline', - ) - try: - import edxval # lint-amnesty, pylint: disable=unused-import - installed_apps += ('edxval',) - except ImportError: - pass - if not settings.configured: - settings.configure( - INSTALLED_APPS=installed_apps, - ) - django.setup() - - try: - root = path(sys.argv[1]) - except IndexError: - sys.exit(main.__doc__) - - descriptor_files = write_descriptor_js(root / 'descriptors/js') - module_files = write_module_js(root / 'modules/js') - write_webpack(root / 'webpack.xmodule.config.js', module_files, descriptor_files) - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/xmodule/template_block.py b/xmodule/template_block.py index a27b92cb2c7e..fd5373386061 100644 --- a/xmodule/template_block.py +++ b/xmodule/template_block.py @@ -6,13 +6,11 @@ from xblock.core import XBlock from lxml import etree -from pkg_resources import resource_filename from web_fragments.fragment import Fragment from xmodule.editing_block import EditingMixin from xmodule.raw_block import RawMixin from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -28,7 +26,6 @@ class CustomTagTemplateBlock( # pylint: disable=abstract-method XmlMixin, EditingMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): @@ -65,15 +62,6 @@ class CustomTagBlock(CustomTagTemplateBlock): # pylint: disable=abstract-method resources_dir = None template_dir_name = 'customtag' - preview_view_js = { - 'js': [], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - studio_view_js = { - 'js': [resource_filename(__name__, 'js/src/raw/edit/xml.js')], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - def studio_view(self, _context): """ Return the studio view. diff --git a/xmodule/tests/test_content.py b/xmodule/tests/test_content.py index dabd6d752739..0b566153ad7d 100644 --- a/xmodule/tests/test_content.py +++ b/xmodule/tests/test_content.py @@ -1,17 +1,14 @@ """Tests for contents""" -import os import unittest from unittest.mock import Mock, patch import ddt from opaque_keys.edx.keys import CourseKey from opaque_keys.edx.locator import AssetLocator, CourseLocator -from path import Path as path from xmodule.contentstore.content import ContentStore, StaticContent, StaticContentStream -from xmodule.static_content import XBLOCK_CLASSES, _write_js SAMPLE_STRING = """ This is a sample string with more than 1024 bytes, the default STREAM_DATA_CHUNK_SIZE @@ -205,13 +202,3 @@ def test_static_content_stream_stream_data_in_range(self): total_length += len(chunck) assert total_length == ((last_byte - first_byte) + 1) - - def test_static_content_write_js(self): - """ - Test that only one filename starts with 000. - """ - output_root = path('common/static/xmodule/descriptors/js') - file_owners = _write_js(output_root, XBLOCK_CLASSES, 'get_studio_view_js') - js_file_paths = {file_path for file_path in sum(list(file_owners.values()), []) if os.path.basename(file_path).startswith('000-')} # lint-amnesty, pylint: disable=line-too-long - assert len(js_file_paths) == 1 - assert 'XModule.Descriptor = (function() {' in open(js_file_paths.pop()).read() diff --git a/xmodule/video_block/video_block.py b/xmodule/video_block/video_block.py index 97c41c2f6412..5bf2b3c28cf1 100644 --- a/xmodule/video_block/video_block.py +++ b/xmodule/video_block/video_block.py @@ -52,7 +52,7 @@ from xmodule.video_block import manage_video_subtitles_save from xmodule.x_module import ( PUBLIC_VIEW, STUDENT_VIEW, - HTMLSnippet, ResourceTemplates, shim_xmodule_js, + ResourceTemplates, shim_xmodule_js, XModuleMixin, XModuleToXBlockMixin, ) from xmodule.xml_block import XmlMixin, deserialize_field, is_pointer_tag, name_to_pathname @@ -121,7 +121,7 @@ @XBlock.needs('mako', 'user') class VideoBlock( VideoFields, VideoTranscriptsMixin, VideoStudioViewHandlers, VideoStudentViewHandlers, - EmptyDataRawMixin, XmlMixin, EditingMixin, XModuleToXBlockMixin, HTMLSnippet, + EmptyDataRawMixin, XmlMixin, EditingMixin, XModuleToXBlockMixin, ResourceTemplates, XModuleMixin, LicenseMixin): """ XML source example: @@ -282,6 +282,9 @@ def public_view(self, context): return fragment def get_html(self, view=STUDENT_VIEW, context=None): # lint-amnesty, pylint: disable=arguments-differ, too-many-statements + """ + Return html for a given view of this block. + """ context = context or {} track_status = (self.download_track and self.track) transcript_download_format = self.transcript_download_format if not track_status else None diff --git a/xmodule/word_cloud_block.py b/xmodule/word_cloud_block.py index 8be08242dbae..6a31da5ef811 100644 --- a/xmodule/word_cloud_block.py +++ b/xmodule/word_cloud_block.py @@ -10,8 +10,6 @@ import json import logging -from pkg_resources import resource_filename - from web_fragments.fragment import Fragment from xblock.core import XBlock from xblock.fields import Boolean, Dict, Integer, List, Scope, String @@ -20,7 +18,6 @@ from xmodule.util.builtin_assets import add_webpack_js_to_fragment, add_sass_to_fragment from xmodule.xml_block import XmlMixin from xmodule.x_module import ( - HTMLSnippet, ResourceTemplates, shim_xmodule_js, XModuleMixin, @@ -49,7 +46,6 @@ class WordCloudBlock( # pylint: disable=abstract-method XmlMixin, EditingMixin, XModuleToXBlockMixin, - HTMLSnippet, ResourceTemplates, XModuleMixin, ): @@ -112,19 +108,6 @@ class WordCloudBlock( # pylint: disable=abstract-method resources_dir = 'assets/word_cloud' template_dir_name = 'word_cloud' - preview_view_js = { - 'js': [ - resource_filename(__name__, 'assets/word_cloud/src/js/word_cloud.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } - - studio_view_js = { - 'js': [ - resource_filename(__name__, 'js/src/raw/edit/metadata-only.js'), - ], - 'xmodule_js': resource_filename(__name__, 'js/src/xmodule.js'), - } studio_js_module_name = "MetadataOnlyEditingDescriptor" mako_template = "widgets/metadata-only-edit.html" diff --git a/xmodule/x_module.py b/xmodule/x_module.py index 3d4a2801cd61..fe7561cccf3d 100644 --- a/xmodule/x_module.py +++ b/xmodule/x_module.py @@ -206,40 +206,6 @@ def create_definition(self, block_type, slug=None): raise NotImplementedError("Specific Modulestores must provide implementations of create_definition") -class HTMLSnippet: - """ - A base class defining an interface for an object that is able to present an - html snippet, along with associated javascript and css - """ - - preview_view_js = {} - studio_view_js = {} - - @classmethod - def get_preview_view_js(cls): - return cls.preview_view_js - - @classmethod - def get_preview_view_js_bundle_name(cls): - return cls.__name__ + 'Display' - - @classmethod - def get_studio_view_js(cls): - return cls.studio_view_js - - @classmethod - def get_studio_view_js_bundle_name(cls): - return cls.__name__ + 'Editor' - - def get_html(self): - """ - Return the html used to display this snippet - """ - raise NotImplementedError( - "get_html() must be provided by specific modules - not present in {}" - .format(self.__class__)) - - def shim_xmodule_js(fragment, js_module_name): """ Set up the XBlock -> XModule shim on the supplied :class:`web_fragments.fragment.Fragment`