diff --git a/setup.cfg b/setup.cfg index ebec2525..3d3602d9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -39,7 +39,6 @@ console_scripts = frontend_deploy.py = tubular.scripts.frontend_deploy:frontend_deploy frontend_multi_build.py = tubular.scripts.frontend_multi_build:frontend_build frontend_multi_deploy.py = tubular.scripts.frontend_multi_deploy:frontend_deploy - get_learners_to_retire.py = tubular.scripts.get_learners_to_retire:get_learners_to_retire get_ready_to_merge_prs.py = tubular.scripts.get_ready_to_merge_prs:get_ready_to_merge_prs jenkins_trigger_build.py = tubular.scripts.jenkins_trigger_build:trigger merge-approved-prs = tubular.scripts.merge_approved_prs:octomerge @@ -50,14 +49,9 @@ console_scripts = push_public_to_private.py = tubular.scripts.push_public_to_private:push_public_to_private purge_cloudflare_cache.py = tubular.scripts.purge_cloudflare_cache:purge_cloudflare_cache restrict_to_stage.py = tubular.scripts.restrict_to_stage:restrict_ami_to_stage - retire_one_learner.py = tubular.scripts.retire_one_learner:retire_learner - retirement_archive_and_cleanup.py = tubular.scripts.retirement_archive_and_cleanup:archive_and_cleanup - retirement_bulk_status_update.py = tubular.scripts.retirement_bulk_status_update:update_statuses - retirement_partner_report.py = tubular.scripts.retirement_partner_report:generate_report retrieve_latest_base_ami.py = tubular.scripts.retrieve_latest_base_ami:retrieve_latest_base_ami retrieve_base_ami.py = tubular.scripts.retrieve_base_ami:retrieve_base_ami rollback_asg.py = tubular.scripts.rollback_asg:rollback - structures.py = tubular.scripts.structures:cli submit_slack_msg.py = tubular.scripts.submit_slack_msg:submit_slack_msg [extras] diff --git a/tubular/edx_api.py b/tubular/edx_api.py index 8ff9f612..1ebbaf76 100644 --- a/tubular/edx_api.py +++ b/tubular/edx_api.py @@ -189,73 +189,6 @@ class LmsApi(BaseApiClient): """ LMS API client with convenience methods for making API calls. """ - @_retry_lms_api() - def learners_to_retire(self, states_to_request, cool_off_days=7, limit=None): - """ - Retrieves a list of learners awaiting retirement actions. - """ - params = { - 'cool_off_days': cool_off_days, - 'states': states_to_request - } - if limit: - params['limit'] = limit - api_url = self.get_api_url('api/user/v1/accounts/retirement_queue') - return self._request('GET', api_url, params=params) - - @_retry_lms_api() - def get_learners_by_date_and_status(self, state_to_request, start_date, end_date): - """ - Retrieves a list of learners in the given retirement state that were - created in the retirement queue between the dates given. Date range - is inclusive, so to get one day you would set both dates to that day. - - :param state_to_request: String LMS UserRetirementState state name (ex. COMPLETE) - :param start_date: Date or Datetime object - :param end_date: Date or Datetime - """ - params = { - 'start_date': start_date.strftime('%Y-%m-%d'), - 'end_date': end_date.strftime('%Y-%m-%d'), - 'state': state_to_request - } - api_url = self.get_api_url('api/user/v1/accounts/retirements_by_status_and_date') - return self._request('GET', api_url, params=params) - - @_retry_lms_api() - def get_learner_retirement_state(self, username): - """ - Retrieves the given learner's retirement state. - """ - api_url = self.get_api_url(f'api/user/v1/accounts/{username}/retirement_status') - return self._request('GET', api_url) - - @_retry_lms_api() - def update_learner_retirement_state(self, username, new_state_name, message, force=False): - """ - Updates the given learner's retirement state to the retirement state name new_string - with the additional string information in message (for logging purposes). - """ - data = { - 'username': username, - 'new_state': new_state_name, - 'response': message - } - - if force: - data['force'] = True - - api_url = self.get_api_url('api/user/v1/accounts/update_retirement_status') - return self._request('PATCH', api_url, json=data) - - @_retry_lms_api() - def retirement_deactivate_logout(self, learner): - """ - Performs the user deactivation and forced logout step of learner retirement - """ - data = {'username': learner['original_username']} - api_url = self.get_api_url('api/user/v1/accounts/deactivate_logout') - return self._request('POST', api_url, json=data) @_retry_lms_api() def retirement_retire_forum(self, learner): @@ -289,26 +222,6 @@ def retirement_unenroll(self, learner): api_url = self.get_api_url('api/enrollment/v1/unenroll') return self._request('POST', api_url, json=data) - # This endpoint additionally returns 500 when the EdxNotes backend service is unavailable. - @_retry_lms_api() - def retirement_retire_notes(self, learner): - """ - Deletes all the user's notes (aka. annotations) - """ - data = {'username': learner['original_username']} - api_url = self.get_api_url('api/edxnotes/v1/retire_user') - return self._request('POST', api_url, json=data) - - @_retry_lms_api() - def retirement_lms_retire_misc(self, learner): - """ - Deletes, blanks, or one-way hashes personal information in LMS as - defined in EDUCATOR-2802 and sub-tasks. - """ - data = {'username': learner['original_username']} - api_url = self.get_api_url('api/user/v1/accounts/retire_misc') - return self._request('POST', api_url, json=data) - @_retry_lms_api() def retirement_lms_retire(self, learner): """ @@ -318,15 +231,6 @@ def retirement_lms_retire(self, learner): api_url = self.get_api_url('api/user/v1/accounts/retire') return self._request('POST', api_url, json=data) - @_retry_lms_api() - def retirement_partner_queue(self, learner): - """ - Calls LMS to add the given user to the retirement reporting queue - """ - data = {'username': learner['original_username']} - api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report') - return self._request('PUT', api_url, json=data) - @_retry_lms_api() def retirement_partner_report(self): """ @@ -344,56 +248,6 @@ def retirement_partner_cleanup(self, usernames): api_url = self.get_api_url('api/user/v1/accounts/retirement_partner_report_cleanup') return self._request('POST', api_url, json=usernames) - @_retry_lms_api() - def retirement_retire_proctoring_data(self, learner): - """ - Deletes or hashes learner data from edx-proctoring - """ - api_url = self.get_api_url(f"api/edx_proctoring/v1/retire_user/{learner['user']['id']}") - return self._request('POST', api_url) - - @_retry_lms_api() - def retirement_retire_proctoring_backend_data(self, learner): - """ - Removes the given learner from 3rd party proctoring backends - """ - api_url = self.get_api_url(f"api/edx_proctoring/v1/retire_backend_user/{learner['user']['id']}") - return self._request('POST', api_url) - - @_retry_lms_api() - def bulk_cleanup_retirements(self, usernames): - """ - Deletes the retirements for all given usernames - """ - data = {'usernames': usernames} - api_url = self.get_api_url('api/user/v1/accounts/retirement_cleanup') - return self._request('POST', api_url, json=data) - - def replace_lms_usernames(self, username_mappings): - """ - Calls LMS API to replace usernames. - - Param: - username_mappings: list of dicts where key is current username and value is new desired username - [{current_un_1: desired_un_1}, {current_un_2: desired_un_2}] - """ - data = {"username_mappings": username_mappings} - api_url = self.get_api_url('api/user/v1/accounts/replace_usernames') - return self._request('POST', api_url, json=data) - - def replace_forums_usernames(self, username_mappings): - """ - Calls the discussion forums API inside of LMS to replace usernames. - - Param: - username_mappings: list of dicts where key is current username and value is new unique username - [{current_un_1: new_un_1}, {current_un_2: new_un_2}] - """ - data = {"username_mappings": username_mappings} - api_url = self.get_api_url('api/discussion/v1/accounts/replace_usernames') - return self._request('POST', api_url, json=data) - - class EcommerceApi(BaseApiClient): """ Ecommerce API client with convenience methods for making API calls. @@ -407,28 +261,6 @@ def retire_learner(self, learner): api_url = self.get_api_url('api/v2/user/retire') return self._request('POST', api_url, json=data) - @_retry_lms_api() - def get_tracking_key(self, learner): - """ - Fetches the ecommerce tracking id used for Segment tracking when - ecommerce doesn't have access to the LMS user id. - """ - api_url = self.get_api_url(f"api/v2/retirement/tracking_id/{learner['original_username']}") - return self._request('GET', api_url)['ecommerce_tracking_id'] - - def replace_usernames(self, username_mappings): - """ - Calls the ecommerce API to replace usernames. - - Param: - username_mappings: list of dicts where key is current username and value is new unique username - [{current_un_1: new_un_1}, {current_un_2: new_un_2}] - """ - data = {"username_mappings": username_mappings} - api_url = self.get_api_url('api/v2/user_management/replace_usernames') - return self._request('POST', api_url, json=data) - - class CredentialsApi(BaseApiClient): """ Credentials API client with convenience methods for making API calls. @@ -442,36 +274,6 @@ def retire_learner(self, learner): api_url = self.get_api_url('user/retire') return self._request('POST', api_url, json=data) - def replace_usernames(self, username_mappings): - """ - Calls the credentials API to replace usernames. - - Param: - username_mappings: list of dicts where key is current username and value is new unique username - [{current_un_1: new_un_1}, {current_un_2: new_un_2}] - """ - data = {"username_mappings": username_mappings} - api_url = self.get_api_url('api/v2/replace_usernames') - return self._request('POST', api_url, json=data) - - -class DiscoveryApi(BaseApiClient): - """ - Discovery API client with convenience methods for making API calls. - """ - - def replace_usernames(self, username_mappings): - """ - Calls the discovery API to replace usernames. - - Param: - username_mappings: list of dicts where key is current username and value is new unique username - [{current_un_1: new_un_1}, {current_un_2: new_un_2}] - """ - data = {"username_mappings": username_mappings} - api_url = self.get_api_url('api/v1/replace_usernames') - return self._request('POST', api_url, json=data) - class DemographicsApi(BaseApiClient): """ diff --git a/tubular/scripts/structures.py b/tubular/scripts/structures.py deleted file mode 100644 index b37ec0d4..00000000 --- a/tubular/scripts/structures.py +++ /dev/null @@ -1,195 +0,0 @@ -#! /usr/bin/env python3 -""" -Script to detect and prune old Structure documents from the "Split" Modulestore -MongoDB (edxapp.modulestore.structures by default). See docstring/help for the -"make_plan" and "prune" commands for more details. -""" - -import logging -import os -import sys - -import click -import click_log - -# Add top-level module path to sys.path before importing tubular code. -sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from tubular.splitmongo import ChangePlan, SplitMongoBackend # pylint: disable=wrong-import-position - -LOG = logging.getLogger('structures') -click_log.basic_config(LOG) - - -@click.group() -@click.option( - '--connection', - default="mongodb://localhost:27017", - help=( - 'Connection string to the target mongo database. This defaults to ' - 'localhost without password (that will work against devstack). ' - 'You may need to use urllib.parse.quote_plus() to percent-escape ' - 'your username and password.' - ) -) -@click.option( - '--database-name', - default='edxapp', - help='Name of the edX Mongo database containing the course structures to prune.' -) -@click.pass_context -def cli(ctx, connection, database_name): - """ - Recover space on MongoDB for edx-platform by deleting unreachable, - historical course content data. To use, first make a change plan with the - "make_plan" command, and then execute that plan against the database with - the "prune" command. - - This script provides logic to clean up old, unused course content data for - the DraftVersioningModuleStore modulestore, more commonly referred to as the - "Split Mongo" or "Split" modulestore (DraftVersioningModuleStore subclasses - SplitMongoModuleStore). All courses and assets that have newer style locator - keys use DraftVersioningModuleStore. These keys start with "course-v1:", - "ccx-v1:", or "block-v1:". Studio authored content data for this modulestore - is saved as immutable data structures. The edx-platform code never cleans up - old data however, meaning there is an unbounded history of a course's - content revisions stored in MongoDB. - - The older modulestore is DraftModuleStore, sometimes called "Old Mongo". - This code does not address that modulestore in any way. That modulestore - handles courses that use the old "/" separator, such as - "MITx/6.002x/2012_Spring", as well as assets starting with "i4x://". - """ - if ctx.obj is None: - ctx.obj = dict() - - ctx.obj['BACKEND'] = SplitMongoBackend(connection, database_name) - - -@cli.command("make_plan") -@click_log.simple_verbosity_option(default='INFO') -@click.argument('plan_file', type=click.File('w')) -@click.option( - '--details', - type=click.File('w'), - default=None, - help="Name of file to write the human-readable details of the Change Plan." -) -@click.option( - '--retain', - default=2, - type=click.IntRange(0, None), - help=("The maximum number of intermediate structures to preserve for any " - "single branch of an active version. This value does not include the " - "active or original structures (those are always preserved). Defaults " - "to 2. Put 0 here if you want to prune as much as possible.") -) -@click.option( - '--delay', - default=15000, - type=click.IntRange(0, None), - help=("Delay in milliseconds between queries to fetch structures from MongoDB " - "during plan creation. Tune to adjust load on the database.") -) -@click.option( - '--batch-size', - default=10000, - type=click.IntRange(1, None), - help="How many Structures do we fetch at a time?" -) -@click.option( - '--ignore-missing/--no-ignore-missing', - default=False, - help=("Force plan creation, even if missing structures are found. " - "Should repair invalid ids by repointing to original. " - "Review of plan highly recommended") -) -@click.option( - '--dump-structures/--no-dump-structures', - default=False, - help="Dump all strucutres to stderr for debugging or recording state before cleanup." -) -@click.pass_context -def make_plan(ctx, plan_file, details, retain, delay, batch_size, ignore_missing, dump_structures): - """ - Create a Change Plan JSON file describing the operations needed to prune the - database. This command is read-only and does not alter the database. - - The Change Plan JSON is a dictionary with two keys: - - "delete" - A sorted array of Structure document IDs to delete. Since MongoDB - object IDs are created in ascending order by timestamp, this means that the - oldest documents come earlier in the list. - - "update_parents" - A list of [Structure ID, New Parent/Previous ID] pairs. - This is used to re-link the oldest preserved Intermediate Structure back to - the Original Structure, so that we don't leave the database in a state where - a Structure's "previous_version" points to a deleted Structure. - - Specifying a --details file will generate a more verbose, human-readable - text description of the Change Plan for verification purposes. The details - file will only display Structures that are reachable from an Active Version, - so any Structures that are "orphaned" as a result of partial runs of this - script or Studio race conditions will not be reflected. That being said, - orphaned Structures are detected and properly noted in the Change Plan JSON. - """ - structures_graph = ctx.obj['BACKEND'].structures_graph(delay / 1000.0, batch_size) - - # This will create the details file as a side-effect, if specified. - change_plan = ChangePlan.create(structures_graph, retain, ignore_missing, dump_structures, details) - change_plan.dump(plan_file) - - -@cli.command() -@click_log.simple_verbosity_option(default='INFO') -@click.argument('plan_file', type=click.File('r')) -@click.option( - '--delay', - default=15000, - type=click.IntRange(0, None), - help=("Delay in milliseconds between batch deletions during pruning. Tune to " - "adjust load on the database.") -) -@click.option( - '--batch-size', - default=1000, - type=click.IntRange(1, None), - help=("How many Structures do we delete at a time? Tune to adjust load on " - "the database.") -) -@click.option( - '--start', - default=None, - help=("Structure ID to start deleting from. Specifying a Structure ID that " - "is not in the Change Plan is an error. Specifying a Structure ID that " - "has already been deleted is NOT an error, so it's safe to re-run.") -) -@click.pass_context -def prune(ctx, plan_file, delay, batch_size, start): - """ - Prune the MongoDB database according to a Change Plan file. - - This command tries to be as safe as possible. It executes parent updates - before deletes, so an interruption at any point should be safe in that it - won't leave the structure graphs in an inconsistent state. It should also - be safe to resume pruning with the same Change Plan in the event of an - interruption. - - It's also safe to run while Studio is still operating, though you should be - careful to test and tweak the delay and batch_size options to throttle load - on your database. - """ - change_plan = ChangePlan.load(plan_file) - if start is not None and start not in change_plan.delete: - raise click.BadParameter( - "{} is not in the Change Plan {}".format( - start, click.format_filename(plan_file.name) - ), - param_hint='--start' - ) - ctx.obj['BACKEND'].update(change_plan, delay / 1000.0, batch_size, start) - - -if __name__ == '__main__': - # pylint doesn't grok click magic, but this is straight from their docs... - cli(obj={}) # pylint: disable=no-value-for-parameter, unexpected-keyword-arg