From bd70184e6330d0f5a44336440f8628153a5fe4a5 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Sun, 3 Dec 2017 16:58:12 -0600 Subject: [PATCH 1/2] Add service accounts and builtin auth --- .../versions/1fdbfd6b0eea_create_instance.py | 3 +- .../versions/3ce1572cbc6b_create_tasks.py | 2 +- .../458762cd0419_create_authn_user.py | 2 +- .../ba0652dfa1be_create_public_keys.py | 4 +- .../versions/dadf4ada480a_create_authz.py | 522 ++++++++++++++---- .../e7d4dba0f699_create_builtin_user.py | 36 ++ ingredients_db/database.py | 18 +- ingredients_db/models/authn.py | 37 +- ingredients_db/models/authz.py | 35 +- ingredients_db/models/builtin.py | 18 + ingredients_db/models/instance.py | 5 +- ingredients_db/models/project.py | 18 +- ingredients_db/models/public_key.py | 2 +- ingredients_db/models/task.py | 4 +- ingredients_db/test/test_migrations.py | 12 +- requirements.txt | 2 + 16 files changed, 571 insertions(+), 149 deletions(-) create mode 100644 ingredients_db/alembic/versions/e7d4dba0f699_create_builtin_user.py create mode 100644 ingredients_db/models/builtin.py diff --git a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py index 40c4e73..aaacf1c 100644 --- a/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py +++ b/ingredients_db/alembic/versions/1fdbfd6b0eea_create_instance.py @@ -29,6 +29,8 @@ def upgrade(): sa.Column('network_port_id', sau.UUIDType, sa.ForeignKey('network_ports.id', ondelete='RESTRICT')), sa.Column('region_id', sau.UUIDType, sa.ForeignKey('regions.id', ondelete='RESTRICT'), nullable=False), sa.Column('zone_id', sau.UUIDType, sa.ForeignKey('zones.id', ondelete='RESTRICT')), + sa.Column('service_account_id', sau.UUIDType, sa.ForeignKey('authn_service_accounts.id', ondelete='RESTRICT'), + nullable=False), sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False), sa.Column('current_task_id', sau.UUIDType, sa.ForeignKey('tasks.id')), @@ -38,7 +40,6 @@ def upgrade(): sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), onupdate=sa.text('clock_timestamp()'), nullable=False), - ) diff --git a/ingredients_db/alembic/versions/3ce1572cbc6b_create_tasks.py b/ingredients_db/alembic/versions/3ce1572cbc6b_create_tasks.py index 5bebf9b..78f68fc 100644 --- a/ingredients_db/alembic/versions/3ce1572cbc6b_create_tasks.py +++ b/ingredients_db/alembic/versions/3ce1572cbc6b_create_tasks.py @@ -27,7 +27,7 @@ def upgrade(): sa.Column('error_message', sa.Text), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - nullable=False), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), onupdate=sa.text('clock_timestamp()'), nullable=False), diff --git a/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py b/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py index ad60a66..d423f31 100644 --- a/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py +++ b/ingredients_db/alembic/versions/458762cd0419_create_authn_user.py @@ -23,7 +23,7 @@ def upgrade(): sa.Column('username', sa.String, nullable=False), sa.Column('driver', sa.String, nullable=False), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - nullable=False), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), onupdate=sa.text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/alembic/versions/ba0652dfa1be_create_public_keys.py b/ingredients_db/alembic/versions/ba0652dfa1be_create_public_keys.py index d6b565c..08c5f99 100644 --- a/ingredients_db/alembic/versions/ba0652dfa1be_create_public_keys.py +++ b/ingredients_db/alembic/versions/ba0652dfa1be_create_public_keys.py @@ -26,7 +26,7 @@ def upgrade(): sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='CASCADE'), nullable=False), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - nullable=False), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), onupdate=sa.text('clock_timestamp()'), nullable=False) @@ -40,7 +40,7 @@ def upgrade(): sa.Column('instance_id', sau.UUIDType, sa.ForeignKey('instances.id', ondelete='CASCADE')), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - nullable=False), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), onupdate=sa.text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py b/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py index 0845992..128fb06 100644 --- a/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py +++ b/ingredients_db/alembic/versions/dadf4ada480a_create_authz.py @@ -10,6 +10,7 @@ from alembic import op # revision identifiers, used by Alembic. + revision = 'dadf4ada480a' down_revision = 'f422a466b0a8' branch_labels = None @@ -21,163 +22,486 @@ def upgrade(): 'authz_policies', sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), sa.Column('name', sa.String, nullable=False, unique=True), - sa.Column('rule', sa.String, nullable=False), sa.Column('description', sa.String), + sa.Column('tags', sa.ARRAY(sa.String)), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - nullable=False), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - onupdate=sa.text('clock_timestamp()'), - nullable=False) + onupdate=sa.text('clock_timestamp()'), nullable=False) ) roles_table = op.create_table( 'authz_roles', sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), - sa.Column('name', sa.String, nullable=False, unique=True), + sa.Column('name', sa.String, nullable=False), + sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='CASCADE')), sa.Column('description', sa.String), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + nullable=False, index=True), + sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + onupdate=sa.text('clock_timestamp()'), nullable=False) + ) + op.create_unique_constraint('uq_name_project_id', 'authz_roles', ['name', 'project_id']) + + role_policies_table = op.create_table( + 'authz_role_policies', + sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), + sa.Column('role_id', sau.UUIDType, sa.ForeignKey('authz_roles.id', ondelete='CASCADE'), index=True, + nullable=False), + sa.Column('policy_id', sau.UUIDType, sa.ForeignKey('authz_policies.id', ondelete='CASCADE'), index=True, nullable=False), + sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - onupdate=sa.text('clock_timestamp()'), - nullable=False) + onupdate=sa.text('clock_timestamp()'), nullable=False) ) op.create_table( - 'authn_tokens', + 'authn_service_accounts', sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), - sa.Column('access_token', sa.String, nullable=False, index=True), - - sa.Column('user_id', sau.UUIDType, sa.ForeignKey('authn_users.id', ondelete='CASCADE'), nullable=False), - sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='CASCADE')), - + sa.Column('name', sa.String, nullable=False), + sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='CASCADE'), nullable=False), + sa.Column('role_id', sau.UUIDType, sa.ForeignKey('authz_roles.id', ondelete='RESTRICT'), nullable=False), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - onupdate=sa.text('clock_timestamp()'), - nullable=False), - sa.Column('expires_at', sau.ArrowType(timezone=True), nullable=False) + onupdate=sa.text('clock_timestamp()'), nullable=False) ) + # TODO: add project_members op.create_table( - 'authn_token_roles', + 'project_members', sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), - - sa.Column('token_id', sau.UUIDType, sa.ForeignKey('authn_tokens.id', ondelete='CASCADE'), nullable=False), - sa.Column('role_id', sau.UUIDType, sa.ForeignKey('authz_roles.id', ondelete='CASCADE'), nullable=False), - + sa.Column('user_id', sau.UUIDType, sa.ForeignKey('authn_users.id', ondelete='CASCADE'), nullable=False, + index=True), + sa.Column('project_id', sau.UUIDType, sa.ForeignKey('projects.id', ondelete='CASCADE'), nullable=False, + index=True), + sa.Column('role_id', sau.UUIDType, sa.ForeignKey('authz_roles.id', ondelete='RESTRICT'), nullable=False, + index=True), sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), nullable=False, index=True), sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), - onupdate=sa.text('clock_timestamp()'), - nullable=False), + onupdate=sa.text('clock_timestamp()'), nullable=False) ) op.bulk_insert( roles_table, [ - {"name": "admin", "description": "Administrator Role"} + { + "name": "admin", + "description": "Administrator Role" + }, + { + "name": "viewer", + "description": "Viewer role; has the ability to view non-project level objects" + } ] ) + # Policy Tags: + # viewer - policies the viewer role should have + # project_member - policies the default project member role should have + # service_account - policies the default project service account role should have op.bulk_insert( policies_table, [ - # Rules - {"name": "is_admin", "rule": "role:admin", "description": "Is the user in the admin role"}, - {"name": "admin_or_member", "rule": "rule:is_admin or project_id:%(project_id)s", - "description": "Is the user in the admin role or a member of the project of the requested object"}, - {"name": "admin_or_self", "rule": "rule:is_admin or user_id:%(user_id)s", - "description": "Is the user in the admin role or the requested object matches the user id"}, - # Policies - # Use role:admin so we don't get locked out if is_admin is changed/deleted - # If the admin role gets deleted... well don't be stupid :p - {"name": "policies:create", "rule": "role:admin", "description": "Ability to create a policy"}, - {"name": "policies:get", "rule": "role:admin", "description": "Ability to get a policy"}, - {"name": "policies:update", "rule": "role:admin", "description": "Ability to update a policy"}, - {"name": "policies:list", "rule": "role:admin", "description": "Ability to list policies"}, - {"name": "policies:delete", "rule": "role:admin", "description": "Ability to delete a policy"}, + { + "name": "policies:create", + "description": "Ability to create a policy", + }, + { + "name": "policies:get", + "description": "Ability to get a policy", + "tags": [ + "viewer" + ] + + }, + { + "name": "policies:update", + "description": "Ability to update a policy" + }, + { + "name": "policies:list", + "description": "Ability to list policies", + "tags": [ + "viewer" + ] + }, + { + "name": "policies:delete", + "description": "Ability to delete a policy" + }, # Roles - # Use role:admin so we don't get locked out if is_admin is changed/deleted - # If the admin role gets deleted... well don't be stupid :p - {"name": "roles:create", "rule": "role:admin", "description": "Ability to create a role"}, - {"name": "roles:get", "rule": "role:admin", "description": "Ability to get a role"}, - {"name": "roles:list", "rule": "role:admin", "description": "Ability to list roles"}, - {"name": "roles:delete", "rule": "role:admin", "description": "Ability to delete a role"}, + { + "name": "roles:create:global", + "description": "Ability to create a global role" + }, + { + "name": "roles:delete:global", + "description": "Ability to delete a global role" + }, + { + "name": "roles:create:project", + "description": "Ability to create a project role", + "tags": [ + "project_member" + ] + }, + { + "name": "roles:delete:project", + "description": "Ability to delete a project role", + "tags": [ + "project_member" + ] + }, + { + "name": "roles:get", + "description": "Ability to get a role", + "tags": [ + "viewer" + ] + }, + { + "name": "roles:list", + "description": "Ability to list roles", + "tags": [ + "viewer" + ] + }, # Regions - {"name": "regions:create", "rule": "role:admin", "description": "Ability to create a region"}, - {"name": "regions:get", "rule": "", "description": "Ability to get a region"}, - {"name": "regions:list", "rule": "", "description": "Ability to list regions"}, - {"name": "regions:delete", "rule": "role:admin", "description": "Ability to delete a region"}, - {"name": "regions:action:schedule", "rule": "role:admin", - "description": "Ability to change the schedule mode of the region"}, + { + "name": "regions:create", + "description": "Ability to create a region" + }, + { + "name": "regions:get", + "description": "Ability to get a region", + "tags": [ + "viewer" + ] + }, + { + "name": "regions:list", + "description": "Ability to list regions", + "tags": [ + "viewer" + ] + }, + { + "name": "regions:delete", + "description": "Ability to delete a region" + }, + { + "name": "regions:action:schedule", + "description": "Ability to change the schedule mode of the region" + }, # Zones - {"name": "zones:create", "rule": "role:admin", "description": "Ability to create a zone"}, - {"name": "zones:get", "rule": "", "description": "Ability to get a zone"}, - {"name": "zones:list", "rule": "", "description": "Ability to list zones"}, - {"name": "zones:delete", "rule": "role:admin", "description": "Ability to delete a zone"}, - {"name": "zones:action:schedule", "rule": "role:admin", - "description": "Ability to change the schedule mode of the zone"}, - - # Tokens - {"name": "tokens:get", "rule": "rule:admin_or_self", "description": "Ability to get a token"}, + { + "name": "zones:create", + "description": "Ability to create a zone" + }, + { + "name": "zones:get", + "description": "Ability to get a zone", + "tags": [ + "viewer" + ] + }, + { + "name": "zones:list", + "description": "Ability to list zones", + "tags": [ + "viewer" + ] + }, + { + "name": "zones:delete", + "description": "Ability to delete a zone" + }, + { + "name": "zones:action:schedule", + "description": "Ability to change the schedule mode of the zone" + }, # Projects - {"name": "projects:create", "rule": "rule:is_admin", "description": "Ability to create a project"}, - {"name": "projects:get", "rule": "", "description": "Ability to get a project"}, - {"name": "projects:list", "rule": "", "description": "Ability to list projects"}, - {"name": "projects:delete", "rule": "rule:is_admin", "description": "Ability to delete a project"}, + { + "name": "projects:create", + "description": "Ability to create a project" + }, + { + "name": "projects:get", + "description": "Ability to get a project", + "tags": [ + "viewer" + ] + }, + { + "name": "projects:list", + "description": "Ability to list projects", + "tags": [ + "viewer" + ] + }, + { + "name": "projects:delete", + "description": "Ability to delete a project" + }, + # TODO: add policies for project:members # Tasks # Images - {"name": "images:create", "rule": "rule:admin_or_member", "description": "Ability to create an image"}, - {"name": "images:create:public", "rule": "rule:is_admin", - "description": "Ability to create a public image"}, - {"name": "images:get", "rule": "rule:admin_or_member", "description": "Ability to get an image"}, - {"name": "images:list", "rule": "rule:admin_or_member", "description": "Ability to list images"}, - {"name": "images:delete", "rule": "rule:admin_or_member", "description": "Ability to delete an image"}, - {"name": "images:action:lock", "rule": "rule:admin_or_member", "description": "Ability to lock an image"}, - {"name": "images:action:unlock", "rule": "rule:admin_or_member", - "description": "Ability to unlock an image"}, + { + "name": "images:create", + "description": "Ability to create an image", + "tags": [ + "project_member" + ] + }, + { + "name": "images:create:public", + "description": "Ability to create a public image" + }, + { + "name": "images:get", + "description": "Ability to get an image", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "images:list", + "description": "Ability to list images", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "images:delete", + "description": "Ability to delete an image", + "tags": [ + "project_member" + ] + }, + { + "name": "images:action:lock", + "description": "Ability to lock an image", + "tags": [ + "project_member" + ] + }, + { + "name": "images:action:unlock", + "description": "Ability to unlock an image", + "tags": [ + "project_member" + ] + }, # Instances - {"name": "instances:create", "rule": "rule:admin_or_member", - "description": "Ability to create an instance"}, - {"name": "instances:get", "rule": "rule:admin_or_member", "description": "Ability to get an instance"}, - {"name": "instances:list", "rule": "rule:admin_or_member", "description": "Ability to list instances"}, - {"name": "instances:delete", "rule": "rule:admin_or_member", - "description": "Ability to delete an instance"}, - {"name": "instances:action:stop", "rule": "rule:admin_or_member", - "description": "Ability to stop an instance"}, - {"name": "instances:action:start", "rule": "rule:admin_or_member", - "description": "Ability to start an instance"}, - {"name": "instances:action:restart", "rule": "rule:admin_or_member", - "description": "Ability to restart an instance"}, - {"name": "instances:action:image", "rule": "rule:admin_or_member", - "description": "Ability to create an image from an instance"}, - {"name": "instances:action:image:public", "rule": "rule:is_admin", - "description": "Ability to create a public image from an instance"}, - {"name": "instances:action:reset_state", "rule": "rule:admin_or_member", - "description": "Ability to reset the state of an instance to error"}, - {"name": "instances:action:reset_state:active", "rule": "rule:is_admin", - "description": "Ability to reset the state of an instance to active"}, + { + "name": "instances:create", + "description": "Ability to create an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:get", + "description": "Ability to get an instance", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "instances:list", + "description": "Ability to list instances", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "instances:delete", + "description": "Ability to delete an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:stop", + "description": "Ability to stop an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:start", + "description": "Ability to start an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:restart", + "description": "Ability to restart an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:image", + "description": "Ability to create an image from an instance", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:image:public", + "description": "Ability to create a public image from an instance" + }, + { + "name": "instances:action:reset_state", + "description": "Ability to reset the state of an instance to error", + "tags": [ + "project_member" + ] + }, + { + "name": "instances:action:reset_state:active", + "description": "Ability to reset the state of an instance to active", + "tags": [ + "project_member" + ] + }, # Networks - {"name": "networks:create", "rule": "rule:is_admin", "description": "Ability to create a network"}, - {"name": "networks:get", "rule": "", "description": "Ability to get a network"}, - {"name": "networks:list", "rule": "", "description": "Ability to list networks"}, - {"name": "networks:delete", "rule": "rule:is_admin", "description": "Ability to delete a network"}, + { + "name": "networks:create", + "description": "Ability to create a network" + }, + { + "name": "networks:get", + "description": "Ability to get a network", + "tags": [ + "viewer" + ] + }, + { + "name": "networks:list", + "description": "Ability to list networks", + "tags": [ + "viewer" + ] + }, + { + "name": "networks:delete", + "description": "Ability to delete a network" + }, - ] + # Service Accounts + { + "name": "service_accounts:create", + "description": "Ability to create a service account", + "tags": [ + "project_member" + ] + }, + { + "name": "service_accounts:get", + "description": "Ability to get a service account", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "service_accounts:list", + "description": "Ability to list service accounts", + "tags": [ + "project_member", + "service_account" + ] + }, + { + "name": "service_accounts:delete", + "description": "Ability to delete a service account", + "tags": [ + "project_member" + ] + }, + + # BuiltIn Users + { + "name": "builtin:users:create", + "description": "Ability to create users", + }, + { + "name": "builtin:users:get", + "description": "Ability to get a user", + "tags": [ + "viewer" + ] + }, + { + "name": "builtin:users:list", + "description": "Ability to list users", + "tags": [ + "viewer" + ] + }, + { + "name": "builtin:users:delete", + "description": "Ability to delete a user" + }, + { + "name": "builtin:users:password", + "description": "Ability to change a user's password" + }, + { + "name": "builtin:users:role:add", + "description": "Ability to add a role to a user" + }, + { + "name": "builtin:users:role:remove", + "description": "Ability to remove a user from a role" + } + + ], + multiinsert=False # Needed so the list insert works correctly ) + connection = op.get_bind() + admin_role = connection.execute(roles_table.select().where(roles_table.c.name == "admin")).fetchone() + viewer_role = connection.execute(roles_table.select().where(roles_table.c.name == "viewer")).fetchone() + for policy in connection.execute(policies_table.select()): + connection.execute( + role_policies_table.insert().values( + role_id=admin_role.id, + policy_id=policy.id + ) + ) + if policy.tags is not None: + if 'viewer' in policy.tags: + connection.execute( + role_policies_table.insert().values( + role_id=viewer_role.id, + policy_id=policy.id + ) + ) + def downgrade(): - op.drop_table('authn_token_roles') - op.drop_table('authn_tokens') + op.drop_table('authn_service_accounts') + op.drop_table('authz_role_policies') + op.drop_table('project_members') op.drop_table('authz_roles') op.drop_table('authz_policies') diff --git a/ingredients_db/alembic/versions/e7d4dba0f699_create_builtin_user.py b/ingredients_db/alembic/versions/e7d4dba0f699_create_builtin_user.py new file mode 100644 index 0000000..fdc4573 --- /dev/null +++ b/ingredients_db/alembic/versions/e7d4dba0f699_create_builtin_user.py @@ -0,0 +1,36 @@ +"""create builtin user + +Revision ID: e7d4dba0f699 +Revises: ba0652dfa1be +Create Date: 2017-12-02 18:13:36.109525 + +""" +import sqlalchemy as sa +import sqlalchemy_utils as sau +from alembic import op +from sqlalchemy.dialects.postgresql import ARRAY + +# revision identifiers, used by Alembic. +revision = 'e7d4dba0f699' +down_revision = 'ba0652dfa1be' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'builtin_users', + sa.Column('id', sau.UUIDType, server_default=sa.text("uuid_generate_v4()"), primary_key=True), + sa.Column('username', sa.String, nullable=False, unique=True), + sa.Column('password', sau.PasswordType(schemes=['bcrypt']), nullable=False), + sa.Column('roles', ARRAY(sa.String), default=list), + sa.Column('created_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + nullable=False, index=True), + sa.Column('updated_at', sau.ArrowType(timezone=True), server_default=sa.text('clock_timestamp()'), + onupdate=sa.text('clock_timestamp()'), + nullable=False) + ) + + +def downgrade(): + op.drop_table('builtin_users') diff --git a/ingredients_db/database.py b/ingredients_db/database.py index ca70f2a..e7f83c7 100644 --- a/ingredients_db/database.py +++ b/ingredients_db/database.py @@ -14,7 +14,9 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker, scoped_session from sqlalchemy.pool import QueuePool, NullPool +from sqlalchemy_utils import force_auto_coercion +force_auto_coercion() Base = declarative_base() @@ -46,6 +48,7 @@ def connect(self): else: self.engine = create_engine(URL(**database), poolclass=QueuePool, pool_size=self.pool_size) self._add_process_guards(self.engine) + self._add_disconnection_guards(self.engine) def _add_process_guards(self, engine): """Add multiprocessing guards. @@ -72,6 +75,19 @@ def checkout(dbapi_connection, connection_record, connection_proxy): connection_record.info['pid'], pid) ) + def _add_disconnection_guards(self, engine): + @sqlalchemy.event.listens_for(engine.pool, "checkout") + def ping_connection(dbapi_connection, connection_record, connection_proxy): + cursor = dbapi_connection.cursor() + try: + cursor.execute("SELECT 1") + except: + connection_proxy._pool.dispose() + # raise DisconnectionError - pool will try + # connecting again up to three times before raising. + raise exc.DisconnectionError() + cursor.close() + @contextmanager def session(self): @@ -79,8 +95,6 @@ def session(self): session.configure(bind=self.engine) session = session() - session.execute('SELECT 1').scalar() - try: yield session finally: diff --git a/ingredients_db/models/authn.py b/ingredients_db/models/authn.py index f23adae..5094957 100644 --- a/ingredients_db/models/authn.py +++ b/ingredients_db/models/authn.py @@ -1,6 +1,6 @@ -import arrow -from sqlalchemy import Column, String, text, ForeignKey, UniqueConstraint -from sqlalchemy_utils import generic_repr, ArrowType, UUIDType +from sqlalchemy import Column, String, text, UniqueConstraint, ForeignKey +from sqlalchemy.ext.declarative import declared_attr +from sqlalchemy_utils import ArrowType, UUIDType from ingredients_db.database import Base @@ -16,34 +16,27 @@ class AuthNUser(Base): username = Column(String, nullable=False) driver = Column(String, nullable=False) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) -@generic_repr -class AuthNToken(Base): - __tablename__ = 'authn_tokens' +# TODO: should we have locked service accounts to prevent accidental deletion/modification? +# i.e the default instance service account +class AuthNServiceAccount(Base): + __tablename__ = 'authn_service_accounts' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) - access_token = Column(String, nullable=False, index=True) - - user_id = Column(UUIDType, ForeignKey('authn_users.id', ondelete='CASCADE'), nullable=False) - project_id = Column(UUIDType, ForeignKey('projects.id', ondelete='CASCADE')) + name = Column(String, nullable=False) + project_id = Column(UUIDType, ForeignKey('projects.id', ondelete='CASCADE'), nullable=False) + role_id = Column(UUIDType, ForeignKey('authz_roles.id', ondelete='RESTRICT'), nullable=False) created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) - expires_at = Column(ArrowType(timezone=True), default=arrow.now().shift(days=+1), nullable=False) - - -class AuthNTokenRole(Base): - __tablename__ = 'authn_token_roles' - id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) - token_id = Column(UUIDType, ForeignKey('authn_tokens.id', ondelete='CASCADE'), nullable=False) - role_id = Column(UUIDType, ForeignKey('authz_roles.id', ondelete='CASCADE'), nullable=False) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) - updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), - onupdate=text('clock_timestamp()'), nullable=False) +class ServiceAccountMixin(object): + @declared_attr + def service_account_id(cls): + return Column(UUIDType, ForeignKey('authn_service_accounts.id', ondelete='RESTRICT'), nullable=False) diff --git a/ingredients_db/models/authz.py b/ingredients_db/models/authz.py index aa4e01b..429e091 100644 --- a/ingredients_db/models/authz.py +++ b/ingredients_db/models/authz.py @@ -1,31 +1,52 @@ -from sqlalchemy import text, Column, String +from sqlalchemy import text, Column, String, ForeignKey, UniqueConstraint, ARRAY from sqlalchemy_utils import UUIDType, ArrowType from ingredients_db.database import Base -# TODO: should we have locked policies to prevent accidental deletion? +# Policies cannot be added/deleted/modified +# These are just for storage class AuthZPolicy(Base): __tablename__ = 'authz_policies' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) name = Column(String, nullable=False, unique=True) - rule = Column(String, nullable=False) description = Column(String) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + # Used to filter policies to create the default project member and service account roles + tags = Column(ARRAY(String)) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) -# TODO: should we have locked roles to prevent accidental deletion? +# TODO: should we have locked roles to prevent accidental deletion/modification? +# i.e default admin role class AuthZRole(Base): __tablename__ = 'authz_roles' + __table_args__ = ( + UniqueConstraint('name', 'project_id', name='uq_name_project_id'), + ) + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) - name = Column(String, nullable=False, unique=True) + name = Column(String, nullable=False) + project_id = Column(UUIDType, ForeignKey('projects.id', ondelete='CASCADE')) description = Column(String) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) + updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), + onupdate=text('clock_timestamp()'), nullable=False) + + +class AuthZRolePolicy(Base): + __tablename__ = 'authz_role_policies' + + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) + role_id = Column(UUIDType, ForeignKey('authz_roles.id', ondelete='CASCADE'), index=True, nullable=False) + policy_id = Column(UUIDType, ForeignKey('authz_policies.id', ondelete='CASCADE'), index=True, nullable=False) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/models/builtin.py b/ingredients_db/models/builtin.py new file mode 100644 index 0000000..aea38cb --- /dev/null +++ b/ingredients_db/models/builtin.py @@ -0,0 +1,18 @@ +from sqlalchemy import String, Column, text +from sqlalchemy.dialects.postgresql import ARRAY +from sqlalchemy_utils import ArrowType, UUIDType, PasswordType + +from ingredients_db.database import Base + + +class BuiltInUser(Base): + __tablename__ = 'builtin_users' + + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) + username = Column(String, nullable=False, unique=True) + password = Column(PasswordType(schemes=['bcrypt']), nullable=False) + roles = Column(ARRAY(String), default=list) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) + updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), + onupdate=text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/models/instance.py b/ingredients_db/models/instance.py index b2e2821..9035af2 100644 --- a/ingredients_db/models/instance.py +++ b/ingredients_db/models/instance.py @@ -6,6 +6,7 @@ from sqlalchemy_utils import UUIDType, generic_repr, ArrowType from ingredients_db.database import Base +from ingredients_db.models.authn import ServiceAccountMixin from ingredients_db.models.network_port import NetworkableMixin from ingredients_db.models.project import ProjectMixin from ingredients_db.models.public_key import PublicKey @@ -28,7 +29,7 @@ class InstanceState(enum.Enum): @generic_repr -class Instance(Base, TaskMixin, NetworkableMixin, ProjectMixin, RegionableNixin, ZonableMixin): +class Instance(Base, TaskMixin, NetworkableMixin, ProjectMixin, RegionableNixin, ZonableMixin, ServiceAccountMixin): __tablename__ = 'instances' id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) @@ -53,6 +54,6 @@ class InstancePublicKey(Base): public_key_id = Column(UUIDType, ForeignKey('public_keys.id', ondelete='CASCADE')) instance_id = Column(UUIDType, ForeignKey('instances.id', ondelete='CASCADE')) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/models/project.py b/ingredients_db/models/project.py index f79466d..c16763e 100644 --- a/ingredients_db/models/project.py +++ b/ingredients_db/models/project.py @@ -14,10 +14,6 @@ class ProjectState(enum.Enum): ERROR = 'ERROR' -# TODO: project members (users) -# only project members can do things within the project - - @generic_repr class Project(Base): __tablename__ = 'projects' @@ -35,3 +31,17 @@ class ProjectMixin(object): @declared_attr def project_id(cls): return Column(UUIDType, ForeignKey('projects.id', ondelete='RESTRICT'), nullable=False) + + +@generic_repr +class ProjectMembers(Base): + __tablename__ = 'project_members' + + id = Column(UUIDType, server_default=text("uuid_generate_v4()"), primary_key=True) + user_id = Column(UUIDType, ForeignKey('authn_users.id', ondelete='CASCADE'), nullable=False, index=True) + role_id = Column(UUIDType, ForeignKey('authz_roles.id', ondelete='RESTRICT'), nullable=False, index=True) + project_id = Column(UUIDType, ForeignKey('projects.id', ondelete='CASCADE'), nullable=False, index=True) + + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) + updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), + onupdate=text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/models/public_key.py b/ingredients_db/models/public_key.py index 8e9b6b6..3ab622b 100644 --- a/ingredients_db/models/public_key.py +++ b/ingredients_db/models/public_key.py @@ -14,6 +14,6 @@ class PublicKey(Base): project_id = Column(UUIDType, ForeignKey('projects.id', ondelete='CASCADE'), nullable=False) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) diff --git a/ingredients_db/models/task.py b/ingredients_db/models/task.py index 8c41d62..d2381a7 100644 --- a/ingredients_db/models/task.py +++ b/ingredients_db/models/task.py @@ -23,7 +23,7 @@ class Task(Base): state = Column(Enum(TaskState), default=TaskState.PENDING, nullable=False) error_message = Column(Text) - created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False) + created_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), nullable=False, index=True) updated_at = Column(ArrowType(timezone=True), server_default=text('clock_timestamp()'), onupdate=text('clock_timestamp()'), nullable=False) stopped_at = Column(ArrowType(timezone=True)) @@ -33,5 +33,3 @@ class TaskMixin(object): @declared_attr def current_task_id(cls): return Column(UUIDType, ForeignKey('tasks.id')) - - # TODO: updated_at doesn't update when child updates. How to fix? diff --git a/ingredients_db/test/test_migrations.py b/ingredients_db/test/test_migrations.py index ec71b94..0cab1d0 100644 --- a/ingredients_db/test/test_migrations.py +++ b/ingredients_db/test/test_migrations.py @@ -74,13 +74,14 @@ def test_model_and_migration_schemas_are_the_same(self, uri_left, uri_right, ale from ingredients_db.models.instance import Instance, InstancePublicKey from ingredients_db.models.network import Network from ingredients_db.models.network_port import NetworkPort - from ingredients_db.models.project import Project + from ingredients_db.models.project import Project, ProjectMembers from ingredients_db.models.public_key import PublicKey from ingredients_db.models.task import Task - from ingredients_db.models.authn import AuthNUser, AuthNToken - from ingredients_db.models.authz import AuthZPolicy, AuthZRole + from ingredients_db.models.authn import AuthNUser, AuthNServiceAccount + from ingredients_db.models.authz import AuthZPolicy, AuthZRole, AuthZRolePolicy from ingredients_db.models.region import Region from ingredients_db.models.zones import Zone + from ingredients_db.models.builtin import BuiltInUser # Make sure the imports don't go away Image.mro() @@ -90,14 +91,17 @@ def test_model_and_migration_schemas_are_the_same(self, uri_left, uri_right, ale Network.mro() NetworkPort.mro() Project.mro() + ProjectMembers.mro() PublicKey.mro() Task.mro() AuthNUser.mro() - AuthNToken.mro() + AuthNServiceAccount.mro() AuthZPolicy.mro() AuthZRole.mro() + AuthZRolePolicy.mro() Region.mro() Zone.mro() + BuiltInUser.mro() setup_extensions(uri_right) prepare_schema_from_models(uri_right, Base) diff --git a/requirements.txt b/requirements.txt index eafd07b..4ad20ab 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,6 @@ sqlalchemy==1.1.14 # MIT sqlalchemy-utils==0.32.16 # BSD-3-Clause alembic==0.9.5 # MIT arrow==0.10.0 # Apache 2.0 +passlib==1.7.1 # BSD-3-Clause +bcrypt==3.1.4 # Apache 2.0 psycopg2 # LGPL \ No newline at end of file From 6447bd72a829e17df6e4954c2ad0d925a16ee514 Mon Sep 17 00:00:00 2001 From: Ryan Belgrave Date: Sun, 3 Dec 2017 17:02:05 -0600 Subject: [PATCH 2/2] fix flake8 on new disconnection guard --- ingredients_db/database.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingredients_db/database.py b/ingredients_db/database.py index e7f83c7..4497b55 100644 --- a/ingredients_db/database.py +++ b/ingredients_db/database.py @@ -81,7 +81,7 @@ def ping_connection(dbapi_connection, connection_record, connection_proxy): cursor = dbapi_connection.cursor() try: cursor.execute("SELECT 1") - except: + except: # noqa: E722 connection_proxy._pool.dispose() # raise DisconnectionError - pool will try # connecting again up to three times before raising.