From ce37a2610fc6a2d8e1fa23a1ac3233f3b9339162 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 17 Aug 2023 11:49:17 -0800 Subject: [PATCH 1/7] Remove the Subscriptions feature --- Makefile | 8 +- apps/api/api-cf.yml.j2 | 10 - .../hyp3_api/api-spec/job_parameters.yml.j2 | 25 - .../src/hyp3_api/api-spec/openapi-spec.yml.j2 | 174 +---- apps/api/src/hyp3_api/handlers.py | 60 +- apps/api/src/hyp3_api/routes.py | 38 +- apps/main-cf.yml.j2 | 51 -- .../src/subscription_manager.py | 42 -- .../subscription-manager-cf.yml.j2 | 112 --- .../src/subscription_worker.py | 109 --- .../subscription-worker-cf.yml.j2 | 114 ---- lib/dynamo/dynamo/__init__.py | 3 +- lib/dynamo/dynamo/jobs.py | 5 +- lib/dynamo/dynamo/subscriptions.py | 99 --- requirements-all.txt | 2 - tests/cfg.env | 1 - tests/conftest.py | 5 - tests/test_api/conftest.py | 4 - tests/test_api/test_api_spec.py | 4 +- tests/test_api/test_list_jobs.py | 31 - tests/test_api/test_subscriptions.py | 636 ------------------ tests/test_dynamo/test_jobs.py | 50 -- tests/test_dynamo/test_subscriptions.py | 414 ------------ tests/test_subscription_manager.py | 29 - tests/test_subscription_worker.py | 357 ---------- 25 files changed, 12 insertions(+), 2371 deletions(-) delete mode 100644 apps/subscription-manager/src/subscription_manager.py delete mode 100644 apps/subscription-manager/subscription-manager-cf.yml.j2 delete mode 100644 apps/subscription-worker/src/subscription_worker.py delete mode 100644 apps/subscription-worker/subscription-worker-cf.yml.j2 delete mode 100644 lib/dynamo/dynamo/subscriptions.py delete mode 100644 tests/test_api/test_subscriptions.py delete mode 100644 tests/test_dynamo/test_subscriptions.py delete mode 100644 tests/test_subscription_manager.py delete mode 100644 tests/test_subscription_worker.py diff --git a/Makefile b/Makefile index e9ea5f3d4..1634b4e95 100644 --- a/Makefile +++ b/Makefile @@ -2,23 +2,19 @@ API = ${PWD}/apps/api/src CHECK_PROCESSING_TIME = ${PWD}/apps/check-processing-time/src GET_FILES = ${PWD}/apps/get-files/src HANDLE_BATCH_EVENT = ${PWD}/apps/handle-batch-event/src -SUBSCRIPTION_MANAGER = ${PWD}/apps/subscription-manager/src -SUBSCRIPTION_WORKER = ${PWD}/apps/subscription-worker/src SCALE_CLUSTER = ${PWD}/apps/scale-cluster/src START_EXECUTION_MANAGER = ${PWD}/apps/start-execution-manager/src START_EXECUTION_WORKER = ${PWD}/apps/start-execution-worker/src UPDATE_DB = ${PWD}/apps/update-db/src UPLOAD_LOG = ${PWD}/apps/upload-log/src DYNAMO = ${PWD}/lib/dynamo -export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SUBSCRIPTION_MANAGER}:${SUBSCRIPTION_WORKER}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} +export PYTHONPATH = ${API}:${CHECK_PROCESSING_TIME}:${GET_FILES}:${HANDLE_BATCH_EVENT}:${SCALE_CLUSTER}:${START_EXECUTION_MANAGER}:${START_EXECUTION_WORKER}:${UPDATE_DB}:${UPLOAD_LOG}:${DYNAMO} build: render python -m pip install --upgrade -r requirements-apps-api.txt -t ${API}; \ python -m pip install --upgrade -r requirements-apps-api-binary.txt --platform manylinux2014_x86_64 --only-binary=:all: -t ${API}; \ python -m pip install --upgrade -r requirements-apps-handle-batch-event.txt -t ${HANDLE_BATCH_EVENT}; \ - python -m pip install --upgrade -r requirements-apps-subscription-manager.txt -t ${SUBSCRIPTION_MANAGER}; \ - python -m pip install --upgrade -r requirements-apps-subscription-worker.txt -t ${SUBSCRIPTION_WORKER}; \ python -m pip install --upgrade -r requirements-apps-scale-cluster.txt -t ${SCALE_CLUSTER}; \ python -m pip install --upgrade -r requirements-apps-start-execution-manager.txt -t ${START_EXECUTION_MANAGER}; \ python -m pip install --upgrade -r requirements-apps-start-execution-worker.txt -t ${START_EXECUTION_WORKER}; \ @@ -45,7 +41,7 @@ render: static: flake8 openapi-validate cfn-lint flake8: - flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,check_processing_time,start_execution_manager,start_execution_worker,update_db,upload_log,dynamo,lambda_logging,subscription_manager,subscription_worker,scale_cluster apps tests lib + flake8 --ignore=E731 --max-line-length=120 --import-order-style=pycharm --statistics --application-import-names hyp3_api,get_files,handle_batch_event,check_processing_time,start_execution_manager,start_execution_worker,update_db,upload_log,dynamo,lambda_logging,scale_cluster apps tests lib openapi-validate: render openapi-spec-validator apps/api/src/hyp3_api/api-spec/openapi-spec.yml diff --git a/apps/api/api-cf.yml.j2 b/apps/api/api-cf.yml.j2 index b867b3975..db2cc819e 100644 --- a/apps/api/api-cf.yml.j2 +++ b/apps/api/api-cf.yml.j2 @@ -6,9 +6,6 @@ Parameters: UsersTable: Type: String - SubscriptionsTable: - Type: String - AuthPublicKey: Type: String @@ -172,12 +169,6 @@ Resources: Action: - dynamodb:GetItem Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${UsersTable}*" - - Effect: Allow - Action: - - dynamodb:GetItem - - dynamodb:PutItem - - dynamodb:Query - Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubscriptionsTable}*" Lambda: Type: AWS::Lambda::Function @@ -186,7 +177,6 @@ Resources: Variables: JOBS_TABLE_NAME: !Ref JobsTable USERS_TABLE_NAME: !Ref UsersTable - SUBSCRIPTIONS_TABLE_NAME: !Ref SubscriptionsTable AUTH_PUBLIC_KEY: !Ref AuthPublicKey AUTH_ALGORITHM: !Ref AuthAlgorithm MONTHLY_JOB_QUOTA_PER_USER: !Ref MonthlyJobQuotaPerUser diff --git a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 index 46a8e2c70..db3d04d42 100644 --- a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 @@ -63,29 +63,4 @@ components: job_parameters: $ref: "#/components/schemas/{{ job_type }}Parameters" - {{ job_type }}Specification: - description: Contains user provided information on creating a new {{ job_type }} subscription, without specific granules. - type: object - additionalProperties: false - required: - - job_type - - name - properties: - job_type: - type: string - enum: - - {{ job_type }} - name: - $ref: "./openapi-spec.yml#components/schemas/name" - job_parameters: - type: object - additionalProperties: false - {% for parameter, parameter_spec in job_spec['parameters'].items() if 'api_schema' in parameter_spec and parameter not in job_spec.get('required_parameters', []) %} - {% if loop.first %} - properties: - {% endif %} - {{ parameter }}: - {{ json.dumps(parameter_spec[ 'api_schema' ]) }} - {% endfor %} - {% endfor %} diff --git a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 index 88dce68fd..82fb9b1e0 100644 --- a/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/openapi-spec.yml.j2 @@ -58,10 +58,6 @@ paths: in: query schema: $ref: "#/components/schemas/start_token" - - name: subscription_id - in: query - schema: - $ref: "#/components/schemas/job_or_subscription_id" responses: "200": @@ -78,7 +74,7 @@ paths: - name: job_id in: path schema: - $ref: "#/components/schemas/job_or_subscription_id" + $ref: "#/components/schemas/job_id" required: true responses: @@ -100,94 +96,6 @@ paths: schema: $ref: "#/components/schemas/user" - /subscriptions: - post: - description: Adds a subscription with search criteria and processing parameters to use to process data as it becomes available - requestBody: - content: - application/json: - schema: - $ref: "#/components/schemas/subscriptions_body" - required: true - responses: - "200": - description: 200 response - content: - application/json: - schema: - type: object - properties: - subscription: - $ref: "#/components/schemas/subscription" - validate_only: - $ref: "#/components/schemas/validate_only" - - - get: - description: Get information about subscriptions for the logged in user - parameters: - - name: name - in: query - schema: - $ref: "#/components/schemas/name" - - name: job_type - in: query - schema: - $ref: "./job_parameters.yml#components/schemas/job_type" - - name: enabled - in: query - schema: - $ref: "#/components/schemas/subscription_enabled" - responses: - "200": - description: 200 response - content: - application/json: - schema: - type: object - required: - - subscriptions - properties: - subscriptions: - $ref: "#/components/schemas/list_of_subscriptions" - /subscriptions/{subscription_id}: - patch: - description: Change information about the given subscription - parameters: - - name: subscription_id - in: path - schema: - $ref: "#/components/schemas/job_or_subscription_id" - required: true - requestBody: - content: - application/json: - schema: - $ref: '#/components/schemas/subscriptions_update_expression' - required: true - responses: - "200": - description: 200 response - content: - application/json: - schema: - $ref: "#/components/schemas/subscription" - get: - description: Get information about a specific subscription - parameters: - - name: subscription_id - in: path - schema: - $ref: "#/components/schemas/job_or_subscription_id" - required: true - responses: - "200": - description: 200 response - content: - application/json: - schema: - $ref: "#/components/schemas/subscription" - components: schemas: @@ -217,74 +125,6 @@ components: next: $ref: "#/components/schemas/next_url" - subscriptions_body: - description: Information about a subscription - type: object - required: - - subscription - additionalProperties: false - properties: - validate_only: - $ref: "#/components/schemas/validate_only" - subscription: - $ref: "#/components/schemas/new_subscription" - - new_subscription: - description: Contains user provided information about a new subscription - type: object - required: - - search_parameters - - job_specification - additionalProperties: false - properties: - search_parameters: - $ref: "#/components/schemas/search_parameters" - job_specification: - $ref: "./job_parameters.yml#components/schemas/job_specification" - - subscriptions_update_expression: - description: Change/update a subscription - type: object - additionalProperties: false - properties: - start: - $ref: "#/components/schemas/datetime" - end: - $ref: "#/components/schemas/datetime" - intersectsWith: - $ref: "#/components/schemas/intersectsWith" - enabled: - $ref: "#/components/schemas/subscription_enabled" - - list_of_subscriptions: - type: array - items: - $ref: "#/components/schemas/subscription" - - subscription: - type: object - required: - - subscription_id - - search_parameters - - job_specification - - user_id - - creation_date - - enabled - additionalProperties: false - properties: - subscription_id: - $ref: "#/components/schemas/job_or_subscription_id" - search_parameters: - $ref: "#/components/schemas/search_parameters" - job_specification: - $ref: "./job_parameters.yml#components/schemas/job_specification" - user_id: - $ref: "#/components/schemas/user_id" - creation_date: - $ref: "#/components/schemas/datetime" - enabled: - $ref: "#/components/schemas/subscription_enabled" - user: description: Information about a user (quota, user id) type: object @@ -410,9 +250,7 @@ components: additionalProperties: false properties: job_id: - $ref: "#/components/schemas/job_or_subscription_id" - subscription_id: - $ref: "#/components/schemas/job_or_subscription_id" + $ref: "#/components/schemas/job_id" user_id: $ref: "#/components/schemas/user_id" job_type: @@ -447,8 +285,8 @@ components: type: boolean default: false - job_or_subscription_id: - description: Unique identifier for a job or subscription + job_id: + description: Unique identifier for a job type: string format: uuid example: 27836b79-e5b2-4d8f-932f-659724ea02c3 @@ -458,10 +296,6 @@ components: type: string example: myUserId - subscription_enabled: - description: Set to false to pause/disable the processing of new data. - type: boolean - datetime: description: Date and time object formatted according to ISO 8601 type: string diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py index 6d8a7fd28..929346ab9 100644 --- a/apps/api/src/hyp3_api/handlers.py +++ b/apps/api/src/hyp3_api/handlers.py @@ -49,14 +49,12 @@ def post_jobs(body, user): return body -def get_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_token=None, - subscription_id=None): +def get_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_token=None): try: start_key = util.deserialize(start_token) if start_token else None except util.TokenDeserializeError: abort(problem_format(400, 'Invalid start_token value')) - jobs, last_evaluated_key = dynamo.jobs.query_jobs(user, start, end, status_code, name, job_type, start_key, - subscription_id) + jobs, last_evaluated_key = dynamo.jobs.query_jobs(user, start, end, status_code, name, job_type, start_key) payload = {'jobs': jobs} if last_evaluated_key is not None: next_token = util.serialize(last_evaluated_key) @@ -92,57 +90,3 @@ def get_user(user): }, 'job_names': get_names_for_user(user) } - - -def post_subscriptions(body, user): - subscription = body['subscription'] - validate_only = body.get('validate_only') - try: - subscription = dynamo.subscriptions.put_subscription(user, subscription, validate_only) - response = { - 'subscription': subscription - } - if validate_only is not None: - response['validate_only'] = validate_only - return response - except ValueError as e: - abort(problem_format(400, str(e))) - - -def get_subscriptions(user, name=None, job_type=None, enabled=None): - subscriptions = dynamo.subscriptions.get_subscriptions_for_user(user, name, job_type, enabled) - payload = { - 'subscriptions': subscriptions - } - return payload - - -def get_subscription_by_id(subscription_id): - subscription = dynamo.subscriptions.get_subscription_by_id(subscription_id) - if subscription is None: - abort(problem_format(404, f'subscription_id does not exist: {subscription_id}')) - return subscription - - -def patch_subscriptions(subscription_id, body, user): - subscription = dynamo.subscriptions.get_subscription_by_id(subscription_id) - if subscription is None: - abort(problem_format(404, f'subscription_id does not exist: {subscription_id}')) - - if subscription['user_id'] != user: - abort(problem_format(403, 'You may not update subscriptions created by a different user')) - - search_parameters = ['start', 'end', 'intersectsWith'] - for key in search_parameters: - if key in body: - subscription['search_parameters'][key] = body[key] - - if 'enabled' in body: - subscription['enabled'] = body['enabled'] - - try: - dynamo.subscriptions.put_subscription(user, subscription) - except ValueError as e: - abort(problem_format(400, str(e))) - - return subscription diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 7ed297ce1..5044a7fcf 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -23,7 +23,7 @@ api_spec = create_spec(api_spec_dict) CORS(app, origins=r'https?://([-\w]+\.)*asf\.alaska\.edu', supports_credentials=True) -AUTHENTICATED_ROUTES = ['/jobs', '/user', '/subscriptions'] +AUTHENTICATED_ROUTES = ['/jobs', '/user'] @app.before_request @@ -142,9 +142,6 @@ def get(self, job_id): parameters = request.openapi.parameters.query start = parameters.get('start') end = parameters.get('end') - subscription_id = parameters.get('subscription_id') - if subscription_id is not None: - subscription_id = str(subscription_id) return jsonify(handlers.get_jobs( parameters.get('user_id') or g.user, start.isoformat(timespec='seconds') if start else None, @@ -153,7 +150,6 @@ def get(self, job_id): parameters.get('name'), parameters.get('job_type'), parameters.get('start_token'), - subscription_id, )) @@ -167,33 +163,6 @@ def get(self): return jsonify(handlers.get_user(g.user)) -class Subscriptions(FlaskOpenAPIView): - def __init__(self, spec): - super().__init__(spec) - self.request_validator = RequestValidator(spec, custom_formatters={'wkt': WKTValidator()}) - self.response_validator = NonValidator - self.openapi_errors_handler = ErrorHandler - - def post(self): - body = request.get_json() - return jsonify(handlers.post_subscriptions(body, g.user)) - - def get(self, subscription_id): - if subscription_id is not None: - return jsonify(handlers.get_subscription_by_id(subscription_id)) - parameters = request.openapi.parameters.query - return jsonify(handlers.get_subscriptions( - g.user, - parameters.get('name'), - parameters.get('job_type'), - parameters.get('enabled'), - )) - - def patch(self, subscription_id): - body = request.get_json() - return jsonify(handlers.patch_subscriptions(subscription_id, body, g.user)) - - app.json_encoder = CustomEncoder jobs_view = Jobs.as_view('jobs', api_spec) @@ -203,8 +172,3 @@ def patch(self, subscription_id): user_view = User.as_view('user', api_spec) app.add_url_rule('/user', view_func=user_view) - -subscriptions_view = Subscriptions.as_view('subscriptions', api_spec) -app.add_url_rule('/subscriptions/', view_func=subscriptions_view, methods=['PATCH', 'GET']) -app.add_url_rule('/subscriptions', view_func=subscriptions_view, methods=['GET'], defaults={'subscription_id': None}) -app.add_url_rule('/subscriptions', view_func=subscriptions_view, methods=['POST']) diff --git a/apps/main-cf.yml.j2 b/apps/main-cf.yml.j2 index b6fc5a263..5b169d273 100644 --- a/apps/main-cf.yml.j2 +++ b/apps/main-cf.yml.j2 @@ -114,7 +114,6 @@ Resources: Parameters: JobsTable: !Ref JobsTable UsersTable: !Ref UsersTable - SubscriptionsTable: !Ref SubscriptionsTable AuthPublicKey: !Ref AuthPublicKey AuthAlgorithm: !Ref AuthAlgorithm MonthlyJobQuotaPerUser: !Ref MonthlyJobQuotaPerUser @@ -157,32 +156,6 @@ Resources: {% endif %} TemplateURL: scale-cluster/scale-cluster-cf.yml - SubscriptionManager: - Type: AWS::CloudFormation::Stack - Properties: - Parameters: - SubscriptionsTable: !Ref SubscriptionsTable - SubscriptionWorkerArn: !GetAtt SubscriptionWorker.Outputs.LambdaArn - {% if security_environment == 'EDC' %} - SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId - SubnetIds: !Join [",", !Ref SubnetIds] - {% endif %} - TemplateURL: subscription-manager/subscription-manager-cf.yml - - SubscriptionWorker: - Type: AWS::CloudFormation::Stack - Properties: - Parameters: - JobsTable: !Ref JobsTable - UsersTable: !Ref UsersTable - SubscriptionsTable: !Ref SubscriptionsTable - MonthlyJobQuotaPerUser: !Ref MonthlyJobQuotaPerUser - {% if security_environment == 'EDC' %} - SecurityGroupId: !GetAtt Cluster.Outputs.SecurityGroupId - SubnetIds: !Join [",", !Ref SubnetIds] - {% endif %} - TemplateURL: subscription-worker/subscription-worker-cf.yml - HandleBatchEvent: Type: AWS::CloudFormation::Stack Properties: @@ -366,27 +339,3 @@ Resources: KeySchema: - AttributeName: user_id KeyType: HASH - - SubscriptionsTable: - Type: AWS::DynamoDB::Table - Properties: - BillingMode: PAY_PER_REQUEST - AttributeDefinitions: - - AttributeName: subscription_id - AttributeType: S - - AttributeName: user_id - AttributeType: S - - AttributeName: creation_date - AttributeType: S - KeySchema: - - AttributeName: subscription_id - KeyType: HASH - GlobalSecondaryIndexes: - - IndexName: user_id - KeySchema: - - AttributeName: user_id - KeyType: HASH - - AttributeName: creation_date - KeyType: RANGE - Projection: - ProjectionType: ALL diff --git a/apps/subscription-manager/src/subscription_manager.py b/apps/subscription-manager/src/subscription_manager.py deleted file mode 100644 index 7701b846f..000000000 --- a/apps/subscription-manager/src/subscription_manager.py +++ /dev/null @@ -1,42 +0,0 @@ -import json -import os - -import boto3 - -import dynamo -from lambda_logging import log_exceptions, logger - -LAMBDA_CLIENT = boto3.client('lambda') - - -def invoke_worker(worker_function_arn: str, subscription: dict) -> dict: - payload = json.dumps( - {'subscription': dynamo.util.convert_decimals_to_numbers(subscription)} - ) - return LAMBDA_CLIENT.invoke( - FunctionName=worker_function_arn, - InvocationType='Event', - Payload=payload, - ) - - -@log_exceptions -def lambda_handler(event, context): - worker_function_arn = os.environ['SUBSCRIPTION_WORKER_ARN'] - logger.info(f'Worker function ARN: {worker_function_arn}') - - subscriptions = dynamo.subscriptions.get_all_subscriptions() - logger.info(f'Got {len(subscriptions)} subscriptions') - - enabled_subscriptions = [subscription for subscription in subscriptions if subscription['enabled']] - logger.info(f'Got {len(enabled_subscriptions)} enabled subscriptions') - - for count, subscription in enumerate(enabled_subscriptions, start=1): - logger.info( - f'({count}/{len(enabled_subscriptions)}) ' - f'Invoking worker for subscription {subscription["subscription_id"]}' - ) - response = invoke_worker(worker_function_arn, subscription) - logger.info( - f'Got response status code {response["StatusCode"]} for subscription {subscription["subscription_id"]}' - ) diff --git a/apps/subscription-manager/subscription-manager-cf.yml.j2 b/apps/subscription-manager/subscription-manager-cf.yml.j2 deleted file mode 100644 index 0cc2cacee..000000000 --- a/apps/subscription-manager/subscription-manager-cf.yml.j2 +++ /dev/null @@ -1,112 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 - -Parameters: - - SubscriptionsTable: - Type: String - - SubscriptionWorkerArn: - Type: String - - {% if security_environment == 'EDC' %} - SecurityGroupId: - Type: String - - SubnetIds: - Type: CommaDelimitedList - {% endif %} - -Resources: - - LogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${Lambda}" - RetentionInDays: 90 - - Role: - Type: {{ 'Custom::JplRole' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::Role' }} - Properties: - {% if security_environment in ('JPL', 'JPL-public') %} - ServiceToken: !ImportValue Custom::JplRole::ServiceToken - Path: /account-managed/hyp3/ - {% endif %} - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - Action: sts:AssumeRole - Principal: - Service: lambda.amazonaws.com - Effect: Allow - ManagedPolicyArns: - - !Ref Policy - {% if security_environment == 'EDC' %} - - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole - {% endif %} - - Policy: - Type: {{ 'Custom::JplPolicy' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::ManagedPolicy' }} - Properties: - {% if security_environment in ('JPL', 'JPL-public') %} - ServiceToken: !ImportValue Custom::JplPolicy::ServiceToken - Path: /account-managed/hyp3/ - {% endif %} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" - - Effect: Allow - Action: - - dynamodb:Scan - Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubscriptionsTable}*" - - Effect: Allow - Action: - - lambda:InvokeFunction - Resource: !Ref SubscriptionWorkerArn - - Lambda: - Type: AWS::Lambda::Function - Properties: - Environment: - Variables: - SUBSCRIPTIONS_TABLE_NAME: !Ref SubscriptionsTable - SUBSCRIPTION_WORKER_ARN: !Ref SubscriptionWorkerArn - Code: src/ - Handler: subscription_manager.lambda_handler - MemorySize: 1024 - Role: !GetAtt Role.Arn - Runtime: python3.9 - Timeout: 900 - {% if security_environment == 'EDC' %} - VpcConfig: - SecurityGroupIds: - - !Ref SecurityGroupId - SubnetIds: !Ref SubnetIds - {% endif %} - - EventInvokeConfig: - Type: AWS::Lambda::EventInvokeConfig - Properties: - FunctionName: !Ref Lambda - Qualifier: $LATEST - MaximumRetryAttempts: 0 - - Schedule: - Type: AWS::Events::Rule - Properties: - ScheduleExpression: "rate(35 minutes)" - Targets: - - Arn: !GetAtt Lambda.Arn - Id: lambda - - EventPermission: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !GetAtt Lambda.Arn - Action: lambda:InvokeFunction - Principal: events.amazonaws.com - SourceArn: !GetAtt Schedule.Arn diff --git a/apps/subscription-worker/src/subscription_worker.py b/apps/subscription-worker/src/subscription_worker.py deleted file mode 100644 index 7b0d24f40..000000000 --- a/apps/subscription-worker/src/subscription_worker.py +++ /dev/null @@ -1,109 +0,0 @@ -from copy import deepcopy -from datetime import datetime, timedelta, timezone - -import asf_search -import dateutil.parser - -import dynamo -from lambda_logging import log_exceptions, logger - - -def get_processed_granules(user_id, name, job_type): - processed_granules = [] - params = {'user': user_id, 'name': name, 'job_type': job_type} - while True: - jobs, next_token = dynamo.jobs.query_jobs(**params) - granules = [job['job_parameters']['granules'][0] for job in jobs] - processed_granules.extend(granules) - if next_token is None: - break - params['start_key'] = next_token - return processed_granules - - -def get_unprocessed_granules(subscription): - processed_granules = get_processed_granules( - subscription['user_id'], - subscription['job_specification']['name'], - subscription['job_specification']['job_type'], - ) - - search_results = asf_search.search(**subscription['search_parameters']) - search_results.raise_if_incomplete() - return [result for result in search_results if result.properties['sceneName'] not in processed_granules] - - -def get_neighbors(granule, depth, platform): - stack = asf_search.baseline_search.stack_from_product(granule) - stack.raise_if_incomplete() - stack = [item for item in stack if - item.properties['temporalBaseline'] < 0 and item.properties['sceneName'].startswith(platform)] - neighbors = [item.properties['sceneName'] for item in stack[-depth:]] - return neighbors - - -def get_jobs_for_granule(subscription, granule): - job_specification = deepcopy(subscription['job_specification']) - if 'job_parameters' not in job_specification: - job_specification['job_parameters'] = {} - job_specification['subscription_id'] = subscription['subscription_id'] - - job_type = job_specification['job_type'] - - if job_type in ['RTC_GAMMA', 'WATER_MAP']: - job_specification['job_parameters']['granules'] = [granule.properties['sceneName']] - payload = [job_specification] - elif job_type in ['AUTORIFT', 'INSAR_GAMMA']: - payload = [] - neighbors = get_neighbors(granule, 2, subscription['search_parameters']['platform']) - for neighbor in neighbors: - job = deepcopy(job_specification) - job['job_parameters']['granules'] = [granule.properties['sceneName'], neighbor] - payload.append(job) - else: - raise ValueError(f'Subscription job type {job_type} not supported') - return payload - - -def get_jobs_for_subscription(subscription, limit): - granules = get_unprocessed_granules(subscription) - jobs = [] - for granule in granules[:limit]: - jobs.extend(get_jobs_for_granule(subscription, granule)) - return jobs - - -def disable_subscription(subscription): - subscription['enabled'] = False - dynamo.subscriptions.put_subscription(subscription['user_id'], subscription) - - -def handle_subscription(subscription): - jobs = get_jobs_for_subscription(subscription, limit=20) - if jobs: - logger.info(f'Submitting {len(jobs)} jobs') - dynamo.jobs.put_jobs(subscription['user_id'], jobs, fail_when_over_quota=False) - - -@log_exceptions -def lambda_handler(event, context) -> None: - subscription = event['subscription'] - logger.info(f'Handling subscription {subscription["subscription_id"]} for user {subscription["user_id"]}') - - if not subscription['enabled']: - raise ValueError(f'subscription {subscription["subscription_id"]} is disabled') - - handle_subscription(subscription) - - cutoff_date = datetime.now(tz=timezone.utc) - timedelta(days=5) - logger.info(f'Cutoff date: {cutoff_date.isoformat()}') - - end_date = dateutil.parser.parse(subscription['search_parameters']['end']) - logger.info(f'Subscription end date: {end_date.isoformat()}') - - unprocessed_granule_count = len(get_unprocessed_granules(subscription)) - logger.info(f'Unprocessed granules: {unprocessed_granule_count}') - - if end_date <= cutoff_date and unprocessed_granule_count == 0: - logger.info(f'Disabling subscription {subscription["subscription_id"]}') - disable_subscription(subscription) diff --git a/apps/subscription-worker/subscription-worker-cf.yml.j2 b/apps/subscription-worker/subscription-worker-cf.yml.j2 deleted file mode 100644 index c1394d102..000000000 --- a/apps/subscription-worker/subscription-worker-cf.yml.j2 +++ /dev/null @@ -1,114 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 - -Parameters: - - SubscriptionsTable: - Type: String - - JobsTable: - Type: String - - UsersTable: - Type: String - - MonthlyJobQuotaPerUser: - Type: Number - - {% if security_environment == 'EDC' %} - SecurityGroupId: - Type: String - - SubnetIds: - Type: CommaDelimitedList - {% endif %} - -Outputs: - - LambdaArn: - Value: !GetAtt Lambda.Arn - -Resources: - - LogGroup: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: !Sub "/aws/lambda/${Lambda}" - RetentionInDays: 90 - - Role: - Type: {{ 'Custom::JplRole' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::Role' }} - Properties: - {% if security_environment in ('JPL', 'JPL-public') %} - ServiceToken: !ImportValue Custom::JplRole::ServiceToken - Path: /account-managed/hyp3/ - {% endif %} - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - Action: sts:AssumeRole - Principal: - Service: lambda.amazonaws.com - Effect: Allow - ManagedPolicyArns: - - !Ref Policy - {% if security_environment == 'EDC' %} - - arn:aws:iam::aws:policy/service-role/AWSLambdaVPCAccessExecutionRole - {% endif %} - - Policy: - Type: {{ 'Custom::JplPolicy' if security_environment in ('JPL', 'JPL-public') else 'AWS::IAM::ManagedPolicy' }} - Properties: - {% if security_environment in ('JPL', 'JPL-public') %} - ServiceToken: !ImportValue Custom::JplPolicy::ServiceToken - Path: /account-managed/hyp3/ - {% endif %} - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - logs:CreateLogStream - - logs:PutLogEvents - Resource: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/lambda/*" - - Effect: Allow - Action: - - dynamodb:PutItem - - dynamodb:Query - Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${JobsTable}*" - - Effect: Allow - Action: dynamodb:GetItem - Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${UsersTable}*" - - Effect: Allow - Action: - - dynamodb:PutItem - Resource: !Sub "arn:aws:dynamodb:${AWS::Region}:${AWS::AccountId}:table/${SubscriptionsTable}*" - - Lambda: - Type: AWS::Lambda::Function - Properties: - Environment: - Variables: - JOBS_TABLE_NAME: !Ref JobsTable - USERS_TABLE_NAME: !Ref UsersTable - SUBSCRIPTIONS_TABLE_NAME: !Ref SubscriptionsTable - MONTHLY_JOB_QUOTA_PER_USER: !Ref MonthlyJobQuotaPerUser - Code: src/ - Handler: subscription_worker.lambda_handler - MemorySize: 1024 - Role: !GetAtt Role.Arn - Runtime: python3.9 - Timeout: 900 - {% if security_environment == 'EDC' %} - VpcConfig: - SecurityGroupIds: - - !Ref SecurityGroupId - SubnetIds: !Ref SubnetIds - {% endif %} - - EventInvokeConfig: - Type: AWS::Lambda::EventInvokeConfig - Properties: - FunctionName: !Ref Lambda - Qualifier: $LATEST - MaximumRetryAttempts: 0 - MaximumEventAgeInSeconds: 240 diff --git a/lib/dynamo/dynamo/__init__.py b/lib/dynamo/dynamo/__init__.py index 81fee39e1..4ddbf3f87 100644 --- a/lib/dynamo/dynamo/__init__.py +++ b/lib/dynamo/dynamo/__init__.py @@ -1,10 +1,9 @@ -from dynamo import jobs, subscriptions, user +from dynamo import jobs, user from dynamo.util import DYNAMODB_RESOURCE __all__ = [ 'DYNAMODB_RESOURCE', 'jobs', - 'subscriptions', 'user', ] diff --git a/lib/dynamo/dynamo/jobs.py b/lib/dynamo/dynamo/jobs.py index 72ad7e609..7281990eb 100644 --- a/lib/dynamo/dynamo/jobs.py +++ b/lib/dynamo/dynamo/jobs.py @@ -98,8 +98,7 @@ def count_jobs(user, start=None, end=None): return job_count -def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None, - subscription_id=None): +def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type=None, start_key=None): table = DYNAMODB_RESOURCE.Table(environ['JOBS_TABLE_NAME']) key_expression = Key('user_id').eq(user) @@ -113,8 +112,6 @@ def query_jobs(user, start=None, end=None, status_code=None, name=None, job_type filter_expression &= Attr('name').eq(name) if job_type is not None: filter_expression &= Attr('job_type').eq(job_type) - if subscription_id is not None: - filter_expression &= Attr('subscription_id').eq(subscription_id) params = { 'IndexName': 'user_id', diff --git a/lib/dynamo/dynamo/subscriptions.py b/lib/dynamo/dynamo/subscriptions.py deleted file mode 100644 index f94f61c94..000000000 --- a/lib/dynamo/dynamo/subscriptions.py +++ /dev/null @@ -1,99 +0,0 @@ -from datetime import datetime, timedelta, timezone -from os import environ -from uuid import uuid4 - -import dateutil.parser -from boto3.dynamodb.conditions import Attr, Key - -from dynamo.util import DYNAMODB_RESOURCE, convert_floats_to_decimals, format_time - - -def validate_subscription(subscription): - start = dateutil.parser.parse(subscription['search_parameters']['start']) - end = dateutil.parser.parse(subscription['search_parameters']['end']) - if end <= start: - raise ValueError(f'End date: {format_time(end)} must be after start date: {format_time(start)}') - - end_threshold_in_days = 180 - max_end = datetime.now(tz=timezone.utc) + timedelta(days=end_threshold_in_days) - if max_end <= end: - raise ValueError(f'End date: {format_time(end)} must be within {end_threshold_in_days} days: ' - f'{format_time(max_end)}') - - job_type = subscription.get('job_specification', {}).get('job_type') - processing_level = subscription.get('search_parameters', {}).get('processingLevel', 'SLC') - if job_type == 'INSAR_GAMMA' and processing_level != 'SLC': - raise ValueError('processingLevel must be SLC when job_type is INSAR_GAMMA') - - -def put_subscription(user, subscription, validate_only=False): - validate_subscription(subscription) - - defaults = { - 'subscription_id': str(uuid4()), - 'creation_date': format_time(datetime.now(tz=timezone.utc)), - 'user_id': user, - 'enabled': True, - } - for key, value in defaults.items(): - if key not in subscription: - subscription[key] = value - - search_defaults = { - 'platform': 'S1', - 'processingLevel': 'SLC', - 'beamMode': ['IW'], - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - } - for key, value in search_defaults.items(): - if key not in subscription['search_parameters']: - subscription['search_parameters'][key] = value - - table = DYNAMODB_RESOURCE.Table(environ['SUBSCRIPTIONS_TABLE_NAME']) - if not validate_only: - table.put_item(Item=convert_floats_to_decimals(subscription)) - return subscription - - -def get_subscriptions_for_user(user, name=None, job_type=None, enabled=None): - table = DYNAMODB_RESOURCE.Table(environ['SUBSCRIPTIONS_TABLE_NAME']) - - filter_expression = Attr('subscription_id').exists() - - if name is not None: - filter_expression &= Attr('job_specification.name').eq(name) - if job_type is not None: - filter_expression &= Attr('job_specification.job_type').eq(job_type) - if enabled is not None: - filter_expression &= Attr('enabled').eq(enabled) - - params = { - 'IndexName': 'user_id', - 'KeyConditionExpression': Key('user_id').eq(user), - 'FilterExpression': filter_expression, - 'ScanIndexForward': False - } - response = table.query(**params) - subscriptions = response['Items'] - while 'LastEvaluatedKey' in response: - params['ExclusiveStartKey'] = response['LastEvaluatedKey'] - response = table.query(**params) - subscriptions.extend(response['Items']) - return subscriptions - - -def get_subscription_by_id(subscription_id): - table = DYNAMODB_RESOURCE.Table(environ['SUBSCRIPTIONS_TABLE_NAME']) - response = table.get_item(Key={'subscription_id': subscription_id}) - return response.get('Item') - - -def get_all_subscriptions(): - table = DYNAMODB_RESOURCE.Table(environ['SUBSCRIPTIONS_TABLE_NAME']) - - response = table.scan() - subscriptions = response['Items'] - while 'LastEvaluatedKey' in response: - response = table.scan(ExclusiveStartKey=response['LastEvaluatedKey']) - subscriptions.extend(response['Items']) - return subscriptions diff --git a/requirements-all.txt b/requirements-all.txt index a8e6ab8f1..ce0ec1ce9 100644 --- a/requirements-all.txt +++ b/requirements-all.txt @@ -1,7 +1,5 @@ -r requirements-apps-api.txt -r requirements-apps-handle-batch-event.txt --r requirements-apps-subscription-manager.txt --r requirements-apps-subscription-worker.txt -r requirements-apps-scale-cluster.txt -r requirements-apps-start-execution-manager.txt -r requirements-apps-start-execution-worker.txt diff --git a/tests/cfg.env b/tests/cfg.env index c85cff093..ee8c72400 100644 --- a/tests/cfg.env +++ b/tests/cfg.env @@ -1,7 +1,6 @@ FLASK_DEBUG=true JOBS_TABLE_NAME=hyp3-db-table-job USERS_TABLE_NAME=hyp3-db-table-user -SUBSCRIPTIONS_TABLE_NAME=hyp3-db-table-subscriptions AUTH_PUBLIC_KEY=123456789 AUTH_ALGORITHM=HS256 MONTHLY_JOB_QUOTA_PER_USER=25 diff --git a/tests/conftest.py b/tests/conftest.py index 37ab564df..247adf92c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -10,7 +10,6 @@ def table_properties(): class TableProperties: jobs_table = get_table_properties_from_template('JobsTable') users_table = get_table_properties_from_template('UsersTable') - subscriptions_table = get_table_properties_from_template('SubscriptionsTable') return TableProperties() @@ -38,10 +37,6 @@ class Tables: TableName=environ['USERS_TABLE_NAME'], **table_properties.users_table, ) - subscriptions_table = DYNAMODB_RESOURCE.create_table( - TableName=environ['SUBSCRIPTIONS_TABLE_NAME'], - **table_properties.subscriptions_table, - ) tables = Tables() yield tables diff --git a/tests/test_api/conftest.py b/tests/test_api/conftest.py index aabe048bf..3aa65b8f0 100644 --- a/tests/test_api/conftest.py +++ b/tests/test_api/conftest.py @@ -10,7 +10,6 @@ AUTH_COOKIE = 'asf-urs' JOBS_URI = '/jobs' USER_URI = '/user' -SUBSCRIPTIONS_URI = '/subscriptions' DEFAULT_JOB_ID = 'myJobId' DEFAULT_USERNAME = 'test_username' @@ -58,7 +57,6 @@ def submit_batch(client, batch=None, validate_only=None): def make_db_record(job_id, - subscription_id=None, granules=None, job_type='RTC_GAMMA', user_id=DEFAULT_USERNAME, @@ -81,8 +79,6 @@ def make_db_record(job_id, 'request_time': request_time, 'status_code': status_code, } - if subscription_id is not None: - record['subscription_id'] = subscription_id if name is not None: record['name'] = name if files is not None: diff --git a/tests/test_api/test_api_spec.py b/tests/test_api/test_api_spec.py index 14da47a64..cee33debb 100644 --- a/tests/test_api/test_api_spec.py +++ b/tests/test_api/test_api_spec.py @@ -1,6 +1,6 @@ from http import HTTPStatus -from test_api.conftest import AUTH_COOKIE, JOBS_URI, SUBSCRIPTIONS_URI, USER_URI, login +from test_api.conftest import AUTH_COOKIE, JOBS_URI, USER_URI, login from hyp3_api import auth, routes @@ -8,8 +8,6 @@ JOBS_URI: {'GET', 'HEAD', 'OPTIONS', 'POST'}, JOBS_URI + '/foo': {'GET', 'HEAD', 'OPTIONS'}, USER_URI: {'GET', 'HEAD', 'OPTIONS'}, - SUBSCRIPTIONS_URI: {'GET', 'HEAD', 'OPTIONS', 'POST'}, - SUBSCRIPTIONS_URI + '/foo': {'GET', 'HEAD', 'OPTIONS', 'PATCH'}, } diff --git a/tests/test_api/test_list_jobs.py b/tests/test_api/test_list_jobs.py index e6fee1f08..06623cc74 100644 --- a/tests/test_api/test_list_jobs.py +++ b/tests/test_api/test_list_jobs.py @@ -181,37 +181,6 @@ def test_list_jobs_date_start_and_end(client, tables): assert response.json == {'jobs': [items[1]]} -def test_list_jobs_subscription_id(client, tables): - items = [ - make_db_record('874f7533-807d-4b20-afe1-27b5b6fc9d6c', subscription_id='9b02d992-e21e-4e2f-9310-5dd469be2708'), - make_db_record('0ddaeb98-7636-494d-9496-03ea4a7df266', subscription_id='9b02d992-e21e-4e2f-9310-5dd469be2708'), - make_db_record('27836b79-e5b2-4d8f-932f-659724ea02c3', subscription_id='9b02d992-e21e-4e2f-9310-5dd469be2700'), - make_db_record('4277c126-6927-4859-b62f-eb3d2a8815c2'), - ] - for item in items: - tables.jobs_table.put_item(Item=item) - - login(client) - response = client.get(JOBS_URI, query_string={'subscription_id': '9b02d992-e21e-4e2f-9310-5dd469be2708'}) - assert response.status_code == HTTPStatus.OK - assert list_have_same_elements(response.json['jobs'], items[:2]) - - response = client.get(JOBS_URI, query_string={'subscription_id': '9b02d992-e21e-4e2f-9310-5dd469be2700'}) - assert response.status_code == HTTPStatus.OK - assert response.json['jobs'] == [items[2]] - - response = client.get(JOBS_URI, query_string={'subscription_id': '55c6981e-c33a-4086-b20b-661ee6f592a9'}) - assert response.status_code == HTTPStatus.OK - assert response.json['jobs'] == [] - - response = client.get(JOBS_URI) - assert response.status_code == HTTPStatus.OK - assert list_have_same_elements(response.json['jobs'], items) - - response = client.get(JOBS_URI, query_string={'subscription_id': 'not a uuid'}) - assert response.status_code == HTTPStatus.BAD_REQUEST - - def test_bad_date_formats(client): datetime_parameters = ['start', 'end'] bad_dates = [ diff --git a/tests/test_api/test_subscriptions.py b/tests/test_api/test_subscriptions.py deleted file mode 100644 index f297f6401..000000000 --- a/tests/test_api/test_subscriptions.py +++ /dev/null @@ -1,636 +0,0 @@ -from datetime import datetime, timedelta, timezone -from http import HTTPStatus - -from .conftest import SUBSCRIPTIONS_URI, login - - -def test_post_subscription(client, tables): - login(client) - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - 'platform': 'S1', - 'beamMode': ['IW'], - 'polarization': ['VV'], - 'processingLevel': 'SLC', - }, - 'job_specification': { - 'job_parameters': { - 'looks': '10x2', - 'include_inc_map': True, - 'include_look_vectors': True, - 'include_displacement_maps': True, - 'include_dem': True, - 'include_wrapped_phase': True, - 'apply_water_mask': True, - }, - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName' - } - } - } - - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.OK - assert 'subscription_id' in response.json['subscription'] - assert 'user_id' in response.json['subscription'] - for k, v in params['subscription'].items(): - assert response.json['subscription'][k] == v - - -def test_submit_subscriptions_missing_fields(client, tables): - login(client) - params = {} - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST - - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST - - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName' - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.OK - - -def test_search_criteria(client, tables): - login(client) - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - 'frame': [50], - 'relativeOrbit': [1, 5], - 'flightDirection': 'ASCENDING', - 'intersectsWith': 'POLYGON((-5 2, -3 2, -3 5, -5 5, -5 2))', - 'processingLevel': 'GRD_HD', - 'polarization': ['VV'], - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.OK - - bad_params = { - 'frame': [99999], - 'relativeOrbit': 123, - 'flightDirection': 'Foo', - 'intersectsWith': '-190,400,200,90', - 'processingLevel': 'OCN', - 'polarization': ['DUAL VV'], - 'undefined': 'foo', - } - for k, v in bad_params.items(): - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - k: v, - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST - - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST - - -def test_get_subscriptions(client, tables): - login(client, username='subscriptionsUser') - response = client.get(SUBSCRIPTIONS_URI) - assert response.status_code == HTTPStatus.OK - assert response.json == {'subscriptions': []} - - items = [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-04T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - }, - { - 'subscription_id': '140191ab-486b-4080-ab1b-3e2c40aab6b8', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-03T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - }, - { - 'subscription_id': '92da7534-1896-410a-99e4-d16a20c71861', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-02T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - ] - for item in items: - tables.subscriptions_table.put_item(Item=item) - response = client.get(SUBSCRIPTIONS_URI) - assert response.json == {'subscriptions': items} - assert response.status_code == HTTPStatus.OK - - -def test_update_subscription(client, tables): - login(client, 'user1') - subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - 'intersectsWith': 'POINT(1 1)', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'subscription_id': 'a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'enabled': True, - } - tables.subscriptions_table.put_item(Item=subscription) - - api_response = client.patch(SUBSCRIPTIONS_URI + '/a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - json={'start': '2020-02-01T00:00:00+00:00', - 'end': '2020-02-02T00:00:00+00:00', - 'intersectsWith': 'POINT(2 2)'}) - assert api_response.status_code == HTTPStatus.OK - assert api_response.json == { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-02-01T00:00:00+00:00', - 'end': '2020-02-02T00:00:00+00:00', - 'intersectsWith': 'POINT(2 2)', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'subscription_id': 'a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'enabled': True, - } - - response = tables.subscriptions_table.scan() - - assert response['Items'][0] == { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-02-01T00:00:00+00:00', - 'end': '2020-02-02T00:00:00+00:00', - 'intersectsWith': 'POINT(2 2)', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'subscription_id': 'a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'enabled': True, - } - - -def test_update_subscription_wrong_user(client, tables): - login(client, 'user1') - - subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'subscription_id': 'a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user2', - 'creation_date': '2020-01-01T00:00:00+00:00', - } - tables.subscriptions_table.put_item(Item=subscription) - api_response = client.patch(SUBSCRIPTIONS_URI + '/a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - json={'end': '2020-02-02T00:00:00+00:00'}) - assert api_response.status_code == HTTPStatus.FORBIDDEN - - -def test_update_subscription_date_too_far_out(client, tables): - login(client, 'user1') - - subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'subscription_id': 'a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1', - 'creation_date': '2020-01-01T00:00:00+00:00', - } - tables.subscriptions_table.put_item(Item=subscription) - - end = datetime.now(tz=timezone.utc) + timedelta(days=181) - api_response = client.patch(SUBSCRIPTIONS_URI + '/a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - json={'end': end}) - assert api_response.status_code == HTTPStatus.BAD_REQUEST - assert f'End date: {end.isoformat(timespec="seconds")} must be within 180 days' in api_response.json['detail'] - - -def test_update_subscription_not_found(client, tables): - login(client, 'subscriptionsUser1') - api_response = client.patch(SUBSCRIPTIONS_URI + '/a97cefdf-1aa7-4bfd-9785-ff93b3e3d621', - json={'end': '2020-01-02T00:00:00+00:00'}) - assert api_response.status_code == HTTPStatus.NOT_FOUND - - -def test_update_enabled(client, tables): - login(client, 'subscriptionsUser') - subscription = { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - tables.subscriptions_table.put_item(Item=subscription) - - response = client.patch(SUBSCRIPTIONS_URI + '/f00b731f-121d-44dc-abfa-c24afd8ad542', json={}) - assert response.json == {'enabled': True, **subscription} - response = tables.subscriptions_table.scan()['Items'][0] - assert response == {'enabled': True, **subscription} - - response = client.patch(SUBSCRIPTIONS_URI + '/f00b731f-121d-44dc-abfa-c24afd8ad542', - json={'enabled': False}) - assert response.json == {'enabled': False, **subscription} - response = tables.subscriptions_table.scan()['Items'][0] - assert response == {'enabled': False, **subscription} - - response = client.patch(SUBSCRIPTIONS_URI + '/f00b731f-121d-44dc-abfa-c24afd8ad542', - json={'enabled': True}) - assert response.json == {'enabled': True, **subscription} - response = tables.subscriptions_table.scan()['Items'][0] - assert response == {'enabled': True, **subscription} - - -def test_get_subscription_by_id(client, tables): - login(client, 'subscriptionsUser1') - items = [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - }, - { - 'subscription_id': '140191ab-486b-4080-ab1b-3e2c40aab6b8', - 'user_id': 'subscriptionsUser2', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName' - } - } - ] - - for item in items: - tables.subscriptions_table.put_item(Item=item) - - response = client.get(SUBSCRIPTIONS_URI + '/f00b731f-121d-44dc-abfa-c24afd8ad542') - assert response.json == items[0] - - response = client.get(SUBSCRIPTIONS_URI + '/140191ab-486b-4080-ab1b-3e2c40aab6b8') - assert response.json == items[1] - - response = client.get(SUBSCRIPTIONS_URI + '/140191ab-486b-4080-ab1b-3e2c40aab6b7') - assert response.status_code == HTTPStatus.NOT_FOUND - - -def test_post_subscriptions_validate_only(client, tables): - login(client, 'subscriptionsUser') - subscription = { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName' - } - } - params = { - 'subscription': { - **subscription - }, - 'validate_only': True, - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - - assert response.status_code == HTTPStatus.OK - - assert tables.subscriptions_table.scan()['Items'] == [] - - params = { - 'subscription': { - **subscription - }, - 'validate_only': False, - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - - assert response.status_code == HTTPStatus.OK - assert len(tables.subscriptions_table.scan()['Items']) == 1 - - -def test_query_subscription_by_name(client, tables): - login(client, 'subscriptionsUser') - items = [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName1' - } - }, - { - 'subscription_id': '140191ab-486b-4080-ab1b-3e2c40aab6b8', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName2' - } - } - ] - - for item in items: - tables.subscriptions_table.put_item(Item=item) - - response = client.get(SUBSCRIPTIONS_URI, query_string={'name': 'SubscriptionName1'}) - assert response.status_code == HTTPStatus.OK - assert response.json['subscriptions'] == [items[0]] - - -def test_query_jobs_by_job_type(client, tables): - login(client, 'subscriptionsUser') - items = [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName1' - } - }, - { - 'subscription_id': '140191ab-486b-4080-ab1b-3e2c40aab6b8', - 'user_id': 'subscriptionsUser', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName1' - } - } - ] - - for item in items: - tables.subscriptions_table.put_item(Item=item) - - response = client.get(SUBSCRIPTIONS_URI, query_string={'job_type': 'RTC_GAMMA'}) - assert response.status_code == HTTPStatus.OK - assert response.json['subscriptions'] == [items[0]] - - -def test_query_jobs_by_enabled(client, tables): - login(client, 'subscriptionsUser') - items = [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'enabled': True, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName1' - } - }, - { - 'subscription_id': '140191ab-486b-4080-ab1b-3e2c40aab6b8', - 'user_id': 'subscriptionsUser', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'enabled': False, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName1' - } - } - ] - - for item in items: - tables.subscriptions_table.put_item(Item=item) - - response = client.get(SUBSCRIPTIONS_URI, query_string={'enabled': True}) - assert response.status_code == HTTPStatus.OK - assert response.json['subscriptions'] == [items[0]] - - response = client.get(SUBSCRIPTIONS_URI, query_string={'enabled': False}) - assert response.status_code == HTTPStatus.OK - assert response.json['subscriptions'] == [items[1]] - - -def test_mixed_subscriptions(client, tables): - login(client) - insar_parameters = { - 'looks': '20x4', - } - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': insar_parameters - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.OK - - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': insar_parameters - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST - - -def test_submit_subscription_with_granules(client): - login(client) - for job_type in ['AUTORIFT', 'INSAR_GAMMA', 'RTC_GAMMA']: - params = { - 'subscription': { - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': job_type, - 'name': 'SubscriptionName', - 'job_parameters': { - 'granules': ['S1B_IW_GRDH_1SDV_20211101T182511_20211101T182536_029398_03822D_2A42'], - }, - } - } - } - response = client.post(SUBSCRIPTIONS_URI, json=params) - assert response.status_code == HTTPStatus.BAD_REQUEST diff --git a/tests/test_dynamo/test_jobs.py b/tests/test_dynamo/test_jobs.py index 00d33a98b..65a8939d9 100644 --- a/tests/test_dynamo/test_jobs.py +++ b/tests/test_dynamo/test_jobs.py @@ -253,56 +253,6 @@ def test_query_jobs_by_type(tables): assert list_have_same_elements(response, table_items[:2]) -def test_query_jobs_by_subscription_id(tables): - table_items = [ - { - 'job_id': 'job1', - 'subscription_id': '9b02d992-e21e-4e2f-9310-5dd469be2708', - 'name': 'name1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job2', - 'subscription_id': '9b02d992-e21e-4e2f-9310-5dd469be2708', - 'name': 'name1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job3', - 'subscription_id': '9b02d992-e21e-4e2f-9310-5dd469be2700', - 'name': 'name1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - { - 'job_id': 'job4', - 'name': 'name1', - 'user_id': 'user1', - 'status_code': 'status1', - 'request_time': '2000-01-01T00:00:00+00:00', - }, - ] - for item in table_items: - tables.jobs_table.put_item(Item=item) - - response, _ = dynamo.jobs.query_jobs('user1', subscription_id='9b02d992-e21e-4e2f-9310-5dd469be2708') - assert list_have_same_elements(response, table_items[:2]) - - response, _ = dynamo.jobs.query_jobs('user1', subscription_id='9b02d992-e21e-4e2f-9310-5dd469be2700') - assert list_have_same_elements(response, [table_items[2]]) - - response, _ = dynamo.jobs.query_jobs('user1', subscription_id='foo') - assert list_have_same_elements(response, []) - - response, _ = dynamo.jobs.query_jobs('user1') - assert list_have_same_elements(response, table_items) - - def test_put_jobs(tables): payload = [ { diff --git a/tests/test_dynamo/test_subscriptions.py b/tests/test_dynamo/test_subscriptions.py deleted file mode 100644 index af3d787d8..000000000 --- a/tests/test_dynamo/test_subscriptions.py +++ /dev/null @@ -1,414 +0,0 @@ -from datetime import datetime, timedelta, timezone -from decimal import Decimal - -import pytest - -import dynamo - - -def test_put_subscription(tables): - subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'job_parameters': { - 'foo': 0.5, - }, - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - } - } - response = dynamo.subscriptions.put_subscription('user1', subscription) - assert [response] == tables.subscriptions_table.scan()['Items'] - - assert 'subscription_id' in response - assert isinstance(response['subscription_id'], str) - del response['subscription_id'] - - assert 'creation_date' in response - assert isinstance(response['creation_date'], str) - del response['creation_date'] - - assert response == { - 'user_id': 'user1', - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'job_parameters': { - 'foo': Decimal('0.5'), - }, - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - }, - 'enabled': True, - } - - -def test_validate_subscription(): - subscription = { - 'search_parameters': { - 'start': '2021-01-01T00:00:00+00:00', - } - } - - good_end_dates = [ - '2021-01-01T00:00:00-00:01', - '2021-01-01T00:01:00+00:00', - dynamo.util.format_time(datetime.now(tz=timezone.utc) + timedelta(days=180)), - ] - - bad_end_dates = [ - '2021-01-01T00:00:00+00:00', - '2021-01-01T00:00:00+00:01', - dynamo.util.format_time(datetime.now(tz=timezone.utc) + timedelta(days=180, seconds=1)), - ] - - for bad_end_date in bad_end_dates: - subscription['search_parameters']['end'] = bad_end_date - with pytest.raises(ValueError): - dynamo.subscriptions.validate_subscription(subscription) - - for good_end_date in good_end_dates: - subscription['search_parameters']['end'] = good_end_date - dynamo.subscriptions.validate_subscription(subscription) - - for good_end_date in good_end_dates: - subscription['search_parameters']['end'] = good_end_date - dynamo.subscriptions.validate_subscription(subscription) - - subscription = { - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'foo', - }, - 'search_parameters': { - 'start': '2021-01-01T00:00:00+00:00', - 'end': '2021-01-02T00:00:00+00:00', - }, - } - dynamo.subscriptions.validate_subscription(subscription) - - subscription['search_parameters']['processingLevel'] = 'SLC' - dynamo.subscriptions.validate_subscription(subscription) - - subscription['search_parameters']['processingLevel'] = 'GRD_HD' - with pytest.raises(ValueError): - dynamo.subscriptions.validate_subscription(subscription) - - -def test_get_subscriptions_for_user(tables): - table_items = [ - { - 'subscription_id': 'sub1', - 'creation_date': '2020-01-04T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub2', - 'creation_date': '2020-01-03T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub3', - 'creation_date': '2020-01-02T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub4', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user2' - }, - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - response = dynamo.subscriptions.get_subscriptions_for_user('user1') - assert response == table_items[:3] - response = dynamo.subscriptions.get_subscriptions_for_user('user2') - assert response == [table_items[3]] - - -def test_get_subscription_by_id(tables): - assert dynamo.subscriptions.get_subscription_by_id('sub1') is None - - table_items = [ - { - 'subscription_id': 'sub1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub2', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user2' - }, - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - - assert dynamo.subscriptions.get_subscription_by_id('sub1') == table_items[0] - assert dynamo.subscriptions.get_subscription_by_id('sub2') == table_items[1] - - -def test_get_all_subscriptions(tables): - table_items = [ - { - 'subscription_id': 'sub1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub2', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub3', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub4', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user2' - }, - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - response = dynamo.subscriptions.get_all_subscriptions() - assert response == table_items - - -def test_put_subscription_update(tables): - subscription = { - 'user_id': 'user1', - 'subscription_id': 'sub1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - } - } - tables.subscriptions_table.put_item(Item=subscription) - - updated_subscription = { - 'creation_date': '2020-01-01T00:00:00+00:00', - 'user_id': 'user1', - 'subscription_id': 'sub1', - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-06-02T00:00:00+00:00', - 'beamMode': ['IW'], - 'platform': 'S1', - 'polarization': ['VV', 'VV+VH', 'HH', 'HH+HV'], - 'processingLevel': 'SLC', - } - } - dynamo.subscriptions.put_subscription('user1', updated_subscription) - - response = tables.subscriptions_table.scan() - assert response['Items'] == [updated_subscription] - - -def test_put_subscription_validate_only(tables): - bad_subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - } - } - with pytest.raises(ValueError): - dynamo.subscriptions.put_subscription('user1', bad_subscription, validate_only=True) - with pytest.raises(ValueError): - dynamo.subscriptions.put_subscription('user1', bad_subscription, validate_only=False) - - good_subscription = { - 'job_definition': { - 'job_type': 'RTC_GAMMA', - 'name': 'sub1', - }, - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-02T00:00:00+00:00', - } - } - dynamo.subscriptions.put_subscription('user1', good_subscription, validate_only=True) - assert tables.subscriptions_table.scan()['Items'] == [] - - dynamo.subscriptions.put_subscription('user1', good_subscription, validate_only=False) - assert tables.subscriptions_table.scan()['Items'] == [good_subscription] - - -def test_query_subscriptions_by_name(tables): - table_items = [ - { - 'job_specification': {'name': 'name1'}, - 'creation_date': '2020-01-04T00:00:00+00:00', - 'subscription_id': 'sub1', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'job_specification': {'name': 'name1'}, - 'creation_date': '2020-01-03T00:00:00+00:00', - 'subscription_id': 'sub2', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'job_specification': {'name': 'name2'}, - 'creation_date': '2020-01-02T00:00:00+00:00', - 'subscription_id': 'sub3', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'job_specification': {'name': 'name1'}, - 'creation_date': '2020-01-01T00:00:00+00:00', - 'subscription_id': 'sub4', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user2' - }, - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - - response = dynamo.subscriptions.get_subscriptions_for_user('user1', name='name1') - assert response == table_items[:2] - - -def test_query_by_active_status(tables): - table_items = [ - { - 'enabled': True, - 'subscription_id': 'sub1', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-04T00:00:00+00:00', - 'user_id': 'user1' - }, - { - 'enabled': True, - 'subscription_id': 'sub2', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-03T00:00:00+00:00', - 'user_id': 'user1' - }, - { - 'enabled': False, - 'subscription_id': 'sub3', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-02T00:00:00+00:00', - 'user_id': 'user1' - } - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - - response = dynamo.subscriptions.get_subscriptions_for_user('user1', enabled=True) - assert response == table_items[:2] - - response = dynamo.subscriptions.get_subscriptions_for_user('user1', enabled=False) - assert response == [table_items[-1]] - - -def test_query_subscriptions_by_job_type(tables): - table_items = [ - { - 'job_specification': {'job_type': 'RTC_GAMMA'}, - 'subscription_id': 'sub1', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-04T00:00:00+00:00', - 'user_id': 'user1' - }, - { - 'job_specification': {'job_type': 'RTC_GAMMA'}, - 'subscription_id': 'sub2', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-03T00:00:00+00:00', - 'user_id': 'user1' - }, - { - 'job_specification': {'job_type': 'INSAR_GAMMA'}, - 'subscription_id': 'sub3', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-02T00:00:00+00:00', - 'user_id': 'user1' - }, - { - 'job_specification': {'job_type': 'AUTORIFT'}, - 'subscription_id': 'sub4', - 'job_type': 'INSAR_GAMMA', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'user_id': 'user2' - }, - ] - for item in table_items: - tables.subscriptions_table.put_item(Item=item) - - response = dynamo.subscriptions.get_subscriptions_for_user('user1', job_type='RTC_GAMMA') - assert response == table_items[:2] - - response = dynamo.subscriptions.get_subscriptions_for_user('user1', job_type='INSAR_GAMMA') - assert response == [table_items[2]] - - -def test_query_subscriptions_sort_order(tables): - table_items = [ - { - 'subscription_id': 'sub1', - 'creation_date': '2020-01-03T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub2', - 'creation_date': '2020-01-02T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - { - 'subscription_id': 'sub3', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1' - }, - ] - for item in [table_items[1], table_items[2], table_items[0]]: - tables.subscriptions_table.put_item(Item=item) - - response = dynamo.subscriptions.get_subscriptions_for_user('user1') - - assert response == table_items diff --git a/tests/test_subscription_manager.py b/tests/test_subscription_manager.py deleted file mode 100644 index af8647b64..000000000 --- a/tests/test_subscription_manager.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -from unittest.mock import call, patch - -import subscription_manager - -TEST_WORKER_ARN = 'test-worker-arn' - - -def test_lambda_handler(tables): - items = [ - {'subscription_id': 'sub1', 'enabled': True}, - {'subscription_id': 'sub2', 'enabled': False}, - {'subscription_id': 'sub3', 'enabled': True}, - {'subscription_id': 'sub4', 'enabled': True}, - ] - for item in items: - tables.subscriptions_table.put_item(Item=item) - - with patch('subscription_manager.invoke_worker') as mock_invoke_worker, \ - patch.dict(os.environ, {'SUBSCRIPTION_WORKER_ARN': TEST_WORKER_ARN}): - mock_invoke_worker.return_value = {'StatusCode': None} - - subscription_manager.lambda_handler(None, None) - - assert mock_invoke_worker.mock_calls == [ - call(TEST_WORKER_ARN, items[0]), - call(TEST_WORKER_ARN, items[2]), - call(TEST_WORKER_ARN, items[3]), - ] diff --git a/tests/test_subscription_worker.py b/tests/test_subscription_worker.py deleted file mode 100644 index 6d8d6a17c..000000000 --- a/tests/test_subscription_worker.py +++ /dev/null @@ -1,357 +0,0 @@ -from datetime import datetime, timedelta, timezone -from unittest.mock import patch - -import asf_search -import pytest - -import subscription_worker - - -def get_asf_product(properties: dict) -> asf_search.ASFProduct: - product = asf_search.ASFProduct() - product.properties.update(properties) - return product - - -def test_get_unprocessed_granules(tables): - items = [ - { - 'job_id': 'job1', - 'request_time': '2021-01-01T00:00:00+00:00', - 'user_id': 'my_user', - 'job_type': 'INSAR_GAMMA', - 'name': 'my_name', - 'job_parameters': { - 'granules': ['processed', 'not_processed'], - }, - }, - { - 'job_id': 'job2', - 'request_time': '2021-01-01T00:00:00+00:00', - 'user_id': 'different_user', - 'job_type': 'INSAR_GAMMA', - 'name': 'subscription1', - 'job_parameters': { - 'granules': ['not_processed', 'not_processed'], - }, - }, - { - 'job_id': 'job3', - 'request_time': '2021-01-01T00:00:00+00:00', - 'user_id': 'user1', - 'job_type': 'INSAR_GAMMA', - 'name': 'different_name', - 'job_parameters': { - 'granules': ['not_processed', 'not_processed'], - }, - }, - { - 'job_id': 'job4', - 'request_time': '2021-01-01T00:00:00+00:00', - 'user_id': 'my_user', - 'job_type': 'RTC_GAMMA', - 'name': 'my_name', - 'job_parameters': { - 'granules': ['not_processed'], - }, - }, - ] - for item in items: - tables.jobs_table.put_item(Item=item) - - subscription = { - 'user_id': 'my_user', - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'my_name', - }, - 'search_parameters': { - 'foo': 'bar', - }, - } - - products = [ - get_asf_product({'sceneName': 'processed'}), - get_asf_product({'sceneName': 'not_processed'}), - ] - - def mock_search(**kwargs) -> asf_search.ASFSearchResults: - assert kwargs == subscription['search_parameters'] - results = asf_search.ASFSearchResults(products) - results.searchComplete = True - return results - - def mock_search_incomplete(**kwargs) -> asf_search.ASFSearchResults: - results = mock_search(**kwargs) - results.searchComplete = False - return results - - with patch('asf_search.search', mock_search): - assert subscription_worker.get_unprocessed_granules(subscription) == products[1:] - - with patch('asf_search.search', mock_search_incomplete), \ - pytest.raises(asf_search.ASFSearchError, match=r'.*Results are incomplete.*'): - subscription_worker.get_unprocessed_granules(subscription) - - -def test_get_processed_granules(): - def mock_job(granule_name: str) -> dict: - return { - 'job_parameters': { - 'granules': [granule_name], - } - } - - def mock_query_jobs(**kwargs): - assert kwargs['user'] == 'my_user' - assert kwargs['name'] == 'my_name' - assert kwargs['job_type'] == 'RTC_GAMMA' - - if 'start_key' not in kwargs: - job = mock_job('granule0') - next_token = 0 - elif kwargs['start_key'] == 0: - job = mock_job('granule1') - next_token = 1 - else: - assert kwargs['start_key'] == 1 - job = mock_job('granule2') - next_token = None - - return [job], next_token - - expected = ['granule0', 'granule1', 'granule2'] - with patch('dynamo.jobs.query_jobs', mock_query_jobs): - assert subscription_worker.get_processed_granules('my_user', 'my_name', 'RTC_GAMMA') == expected - - -def test_get_neighbors(): - granule = get_asf_product({'sceneName': 'granule'}) - - def mock_stack_from_product(payload) -> asf_search.ASFSearchResults: - assert payload == granule - results = asf_search.ASFSearchResults([ - get_asf_product({'sceneName': 'S1A_A', 'temporalBaseline': -3}), - get_asf_product({'sceneName': 'S1B_B', 'temporalBaseline': -2}), - get_asf_product({'sceneName': 'S1A_C', 'temporalBaseline': -1}), - get_asf_product({'sceneName': 'S1B_D', 'temporalBaseline': 0}), - get_asf_product({'sceneName': 'S1A_E', 'temporalBaseline': 1}), - get_asf_product({'sceneName': 'S1B_F', 'temporalBaseline': 2}), - get_asf_product({'sceneName': 'S1A_G', 'temporalBaseline': 3}), - ]) - results.searchComplete = True - return results - - def mock_stack_from_product_incomplete(payload) -> asf_search.ASFSearchResults: - results = mock_stack_from_product(payload) - results.searchComplete = False - return results - - with patch('asf_search.baseline_search.stack_from_product', mock_stack_from_product): - neighbors = subscription_worker.get_neighbors(granule, 1, 'S1') - assert neighbors == ['S1A_C'] - - neighbors = subscription_worker.get_neighbors(granule, 2, 'S1') - assert neighbors == ['S1B_B', 'S1A_C'] - - neighbors = subscription_worker.get_neighbors(granule, 2, 'S1A') - assert neighbors == ['S1A_A', 'S1A_C'] - - neighbors = subscription_worker.get_neighbors(granule, 5, 'S1B') - assert neighbors == ['S1B_B'] - - with patch('asf_search.baseline_search.stack_from_product', mock_stack_from_product_incomplete), \ - pytest.raises(asf_search.ASFSearchError, match=r'.*Results are incomplete.*'): - subscription_worker.get_neighbors(granule, 1, 'S1') - - -def test_get_jobs_for_granule(): - granule = get_asf_product({'sceneName': 'GranuleName'}) - granule2 = get_asf_product({'sceneName': 'GranuleName2'}) - - subscription = { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'user_id': 'subscriptionsUser', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - }, - 'job_specification': { - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': { - 'speckle_filter': True, - } - } - } - payload = subscription_worker.get_jobs_for_granule(subscription, granule) - payload2 = subscription_worker.get_jobs_for_granule(subscription, granule2) - assert payload == [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': { - 'granules': ['GranuleName'], - 'speckle_filter': True, - }, - } - ] - assert payload2 == [ - { - 'subscription_id': 'f00b731f-121d-44dc-abfa-c24afd8ad542', - 'job_type': 'RTC_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': { - 'granules': ['GranuleName2'], - 'speckle_filter': True, - }, - } - ] - - subscription = { - 'subscription_id': '51b576b0-a89b-4108-a9d8-7ecb52aee950', - 'user_id': 'subscriptionsUser', - 'search_parameters': { - 'start': '2020-01-01T00:00:00+00:00', - 'end': '2020-01-01T00:00:00+00:00', - 'platform': 'foo', - }, - 'job_specification': { - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName' - } - } - mock_granules = ['granule2', 'granule3'] - with patch('subscription_worker.get_neighbors', lambda x, y, z: mock_granules): - payload = subscription_worker.get_jobs_for_granule(subscription, granule) - assert payload == [ - { - 'subscription_id': '51b576b0-a89b-4108-a9d8-7ecb52aee950', - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': { - 'granules': ['GranuleName', 'granule2'], - } - }, - { - 'subscription_id': '51b576b0-a89b-4108-a9d8-7ecb52aee950', - 'job_type': 'INSAR_GAMMA', - 'name': 'SubscriptionName', - 'job_parameters': { - 'granules': ['GranuleName', 'granule3'], - }, - } - ] - - subscription = { - 'subscription_id': '51b576b0-a89b-4108-a9d8-7ecb52aee950', - 'job_specification': { - 'job_type': 'FOO', - } - } - with pytest.raises(ValueError): - subscription_worker.get_jobs_for_granule(subscription, granule) - - -def test_get_jobs_for_subscription(): - def mock_get_unprocessed_granules(subscription): - assert subscription == {} - return ['a', 'b', 'c'] - - def mock_get_jobs_for_granule(subscription, granule): - return [{'granule': granule}] - - with patch('subscription_worker.get_unprocessed_granules', mock_get_unprocessed_granules): - with patch('subscription_worker.get_jobs_for_granule', mock_get_jobs_for_granule): - - result = subscription_worker.get_jobs_for_subscription(subscription={}, limit=20) - assert result == [ - {'granule': 'a'}, - {'granule': 'b'}, - {'granule': 'c'}, - ] - - result = subscription_worker.get_jobs_for_subscription(subscription={}, limit=1) - assert result == [ - {'granule': 'a'}, - ] - - result = subscription_worker.get_jobs_for_subscription(subscription={}, limit=0) - assert result == [] - - -def test_lambda_handler(tables): - items = [ - { - 'subscription_id': 'sub1', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'search_parameters': { - 'start': datetime.now(tz=timezone.utc).isoformat(timespec='seconds'), - 'end': (datetime.now(tz=timezone.utc) + timedelta(days=5)).isoformat(timespec='seconds'), - }, - 'user_id': 'user1', - 'enabled': True, - }, - { - 'subscription_id': 'sub2', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'user_id': 'user1', - 'enabled': False - }, - { - 'subscription_id': 'sub3', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'search_parameters': { - 'start': (datetime.now(tz=timezone.utc) - timedelta(days=15)).isoformat(timespec='seconds'), - 'end': (datetime.now(tz=timezone.utc) - timedelta(days=5)).isoformat(timespec='seconds'), - }, - 'user_id': 'user1', - 'enabled': True, - }, - { - 'subscription_id': 'sub4', - 'creation_date': '2020-01-01T00:00:00+00:00', - 'job_type': 'INSAR_GAMMA', - 'search_parameters': { - 'start': (datetime.now(tz=timezone.utc) - timedelta(days=15)).isoformat(timespec='seconds'), - 'end': (datetime.now(tz=timezone.utc) - timedelta(days=5)).isoformat(timespec='seconds'), - }, - 'user_id': 'user1', - 'enabled': True, - }, - ] - for item in items: - tables.subscriptions_table.put_item(Item=item) - - def mock_get_unprocessed_granules(subscription): - if subscription['subscription_id'] == 'sub4': - return ['notempty'] - else: - return [] - - with patch('subscription_worker.handle_subscription') as mock_handle_subscription: - with patch('subscription_worker.get_unprocessed_granules', mock_get_unprocessed_granules): - subscription_worker.lambda_handler({'subscription': items[0]}, None) - - with pytest.raises(ValueError, match=r'subscription sub2 is disabled'): - subscription_worker.lambda_handler.__wrapped__({'subscription': items[1]}, None) - - subscription_worker.lambda_handler({'subscription': items[2]}, None) - subscription_worker.lambda_handler({'subscription': items[3]}, None) - - assert mock_handle_subscription.call_count == 3 - - response = tables.subscriptions_table.scan()['Items'] - - sub1 = [sub for sub in response if sub['subscription_id'] == 'sub1'][0] - assert sub1['enabled'] is True - - sub3 = [sub for sub in response if sub['subscription_id'] == 'sub3'][0] - assert sub3['enabled'] is False - - sub4 = [sub for sub in response if sub['subscription_id'] == 'sub4'][0] - assert sub4['enabled'] is True From 59bc26445ca0eb75c6b3ecc5246713b4516e01ba Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 17 Aug 2023 12:01:16 -0800 Subject: [PATCH 2/7] unused import --- apps/api/src/hyp3_api/routes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 5044a7fcf..2f7f8b071 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -12,7 +12,6 @@ from openapi_core.contrib.flask.handlers import FlaskOpenAPIErrorsHandler from openapi_core.contrib.flask.views import FlaskOpenAPIView from openapi_core.spec.shortcuts import create_spec -from openapi_core.validation.request.validators import RequestValidator from openapi_core.validation.response.datatypes import ResponseValidationResult from hyp3_api import app, auth, handlers From 7d0ca1b3c9dbe9695b3764f0493da5f9518b306b Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Thu, 17 Aug 2023 12:01:54 -0800 Subject: [PATCH 3/7] delete subscription requirements files --- requirements-apps-subscription-manager.txt | 2 -- requirements-apps-subscription-worker.txt | 3 --- 2 files changed, 5 deletions(-) delete mode 100644 requirements-apps-subscription-manager.txt delete mode 100644 requirements-apps-subscription-worker.txt diff --git a/requirements-apps-subscription-manager.txt b/requirements-apps-subscription-manager.txt deleted file mode 100644 index 321852c8c..000000000 --- a/requirements-apps-subscription-manager.txt +++ /dev/null @@ -1,2 +0,0 @@ -./lib/dynamo/ -./lib/lambda_logging/ diff --git a/requirements-apps-subscription-worker.txt b/requirements-apps-subscription-worker.txt deleted file mode 100644 index 117474a1a..000000000 --- a/requirements-apps-subscription-worker.txt +++ /dev/null @@ -1,3 +0,0 @@ -asf-search==6.0.2 -./lib/dynamo/ -./lib/lambda_logging/ From 807be6e85db9836b49692df12157a24d1bd96902 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 18 Aug 2023 09:24:56 -0800 Subject: [PATCH 4/7] delete job_specification schema --- apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 | 6 ------ 1 file changed, 6 deletions(-) diff --git a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 index db3d04d42..694008afb 100644 --- a/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 +++ b/apps/api/src/hyp3_api/api-spec/job_parameters.yml.j2 @@ -21,12 +21,6 @@ components: - $ref: "#/components/schemas/{{ job_type }}Parameters" {% endfor %} - job_specification: - oneOf: - {% for job_type in job_types %} - - $ref: "#/components/schemas/{{ job_type }}Specification" - {% endfor %} - {% for job_type, job_spec in job_types.items() %} {{ job_type }}Parameters: description: Parameters for running {{ job_type }} jobs, including specific granules From 8846a4cba958e435b1c1bc0d05dc90c914ff722a Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 18 Aug 2023 09:27:58 -0800 Subject: [PATCH 5/7] Delete WKTValidator class --- apps/api/src/hyp3_api/routes.py | 12 ------------ tests/test_api/test_api_spec.py | 7 ------- 2 files changed, 19 deletions(-) diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index 2f7f8b071..ca5d63a36 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -103,18 +103,6 @@ def validate(self, res): return ResponseValidationResult() -class WKTValidator: - def validate(self, value): - try: - shapely.wkt.loads(value) - except shapely.errors.WKTReadingError: - return False - return True - - def unmarshal(self, value): - return value - - class ErrorHandler(FlaskOpenAPIErrorsHandler): def __init__(self): super().__init__() diff --git a/tests/test_api/test_api_spec.py b/tests/test_api/test_api_spec.py index cee33debb..087ac6aa9 100644 --- a/tests/test_api/test_api_spec.py +++ b/tests/test_api/test_api_spec.py @@ -110,13 +110,6 @@ def test_ui_location(client): assert response.status_code == HTTPStatus.OK -def test_wkt_validator(client): - validator = routes.WKTValidator() - - assert not validator.validate('foo') - assert validator.validate('POLYGON((-5 2, -3 2, -3 5, -5 5, -5 2))') - - def test_error_format(client): response = client.post(JOBS_URI) assert response.status_code == HTTPStatus.UNAUTHORIZED From 9e6a316143198a144e01a3b38f9d989ed4835184 Mon Sep 17 00:00:00 2001 From: Jake Herrmann Date: Fri, 18 Aug 2023 09:42:45 -0800 Subject: [PATCH 6/7] unused imports --- apps/api/src/hyp3_api/routes.py | 2 -- tests/test_api/test_api_spec.py | 2 +- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/apps/api/src/hyp3_api/routes.py b/apps/api/src/hyp3_api/routes.py index ca5d63a36..829b51f08 100644 --- a/apps/api/src/hyp3_api/routes.py +++ b/apps/api/src/hyp3_api/routes.py @@ -4,8 +4,6 @@ from os import environ from pathlib import Path -import shapely.errors -import shapely.wkt import yaml from flask import abort, g, jsonify, make_response, redirect, render_template, request from flask_cors import CORS diff --git a/tests/test_api/test_api_spec.py b/tests/test_api/test_api_spec.py index 087ac6aa9..20d45630f 100644 --- a/tests/test_api/test_api_spec.py +++ b/tests/test_api/test_api_spec.py @@ -2,7 +2,7 @@ from test_api.conftest import AUTH_COOKIE, JOBS_URI, USER_URI, login -from hyp3_api import auth, routes +from hyp3_api import auth ENDPOINTS = { JOBS_URI: {'GET', 'HEAD', 'OPTIONS', 'POST'}, From 0be51e2002c878e9cfd32174b73dc3755fc3e628 Mon Sep 17 00:00:00 2001 From: Andrew Johnston Date: Fri, 1 Sep 2023 11:55:21 -0800 Subject: [PATCH 7/7] update changelog for v4.0.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9857392c..33752860e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [4.0.0] +### Removed +- The Subscriptions feature has been removed. + - Removed the `/subscriptions` API endpoint. + - Removed the `subscription_id` query parameter from the `GET /jobs` API endpoint. + - Removed the `subscription_id` field from the response body of the `GET /jobs` API endpoint. + ## [3.10.10] ### Changed - Reduced vCPU limits for `hyp3-tibet-jpl` to 0 from 10,000.