heat_template.yaml

heat_template_version: 2017-09-01

description: Heat template to deploy an instance of the Lasair system

parameters:
  key_name:
    description: Name of a KeyPair to enable SSH access.
    type: string
  flavor_small:
    description: Flavor to use for small instances.
    type: string
    default: small
  flavor_medium:
    description: Flavor to use for medium instances.
    type: string
    default: medium
  flavor_large:
    description: Flavor to use for large instances.
    type: string
    default: large
  flavor_xlarge:
    description: Flavor to use for xlarge instances.
    type: string
    default: xlarge
  flavor_kafka:
    description: Flavor to use for kafka instances.
    type: string
    default: kafka 
  flavor_database:
    description: Flavor to use for database instances.
    type: string
    default: database-large
  flavor_parallel_ssh:
    description: Flavor to use for parallel_ssh instances.
    type: string
    default: large
  volume_large:
    description: Size of large volumes
    type: number
    default: 50
  volume_small:
    description: Size of small volumes
    type: number
    default: 25
  image_id:
    description: Name or ID of the image to use.
    type: string
    default: ubuntu-focal-20.04-nogui
  net:
    description: Name of the private network.
    type: string
    default: LSST - Private Network
  external_net:
    description: Name of the external network.
    type: string
    default: external
  share_net:
    description: Name of the network for Manila shares.
    type: string
    default: cephfs
  share_size:
    description: Size of shared filesystem in GB.
    type: number
    default: 10
  share_user:
    description: Username for CephFS.
    type: string
    default: lasair
  fast_volume_type:
    description: Type to use for fast extra volumes
    type: string
    default: ceph-ssd
  slow_volume_type:
    description: Type to use for slow extra volumes
    type: string
    default: ceph-hdd
  db_volume_size:
    description: Size of database extra filesystems in GB.
    type: number
    default: 5
  sherlock_volume_size:
    description: Size of sherlock extra filesystems in GB.
    type: number
    default: 5
  frontend_db_volume_size:
    description: Size of database extra filesystems in GB.
    type: number
    default: 5
  backend_db_volume_size:
    description: Size of database extra filesystems in GB.
    type: number
    default: 5
  cassandra_volume_size:
    description: Size of Cassandra extra filesystems in GB.
    type: number
    default: 15
  kafka_pub_size:
    description: Size of kafka_pub extra filesystems in GB.
    type: number
    default: 15
  n_kafka:
    description: Number of kafka instances
    type: number
    default: 1
  n_ingest:
    description: Number of ingest instances
    type: number
    default: 1
  n_sherlock:
    description: Number of sherlock instances
    type: number
    default: 1
  n_filter:
    description: Number of filter instances
    type: number
    default: 1
  n_cassandra:
    description: Number of cassandra instances
    type: number
    default: 1
  n_frontend_db:
    description: Number of frontend_db instances
    type: number
    default: 0 
  n_backend_db:
    description: Number of backend_db instances
    type: number
    default: 0
  n_parallel_ssh:
    description: Number of parallel_ssh instances
    type: number
    default: 0
  create_db:
    description: Create a standalone database instance
    type: boolean
    default: true
  create_cluster_control:
    description: Create a database cluster control instance
    type: boolean
    default: false
  create_svc:
    description: Create service instance
    type: boolean
    default: true
  create_web:
    description: Create web instance
    type: boolean
    default: true
  create_kafka_pub:
    description: Create kafka_pub instance
    type: boolean
    default: true

conditions:
  create_svc: { get_param: create_svc }
  create_web: { get_param: create_web }
  create_db: { get_param: create_db }
  create_cluster_control: { get_param: create_cluster_control }
  create_kafka_pub: { get_param: create_kafka_pub }

resources:
  default_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "default"] ] }
      policies: ["affinity"]
  share_fs:
    type: OS::Manila::Share
    properties:
      name: { get_param: OS::stack_name }
      description: "Lasair shared filesystem"
      access_rules: [{"access_level": "rw", "access_type": "cephx", "access_to": { get_param: share_user} }]
      size: { get_param: share_size }
      share_protocol: "CEPHFS"
      share_type: "default"
  svc:
    condition: create_svc
    type: server_with_volume.yaml
    properties:
      key_name: { get_param: key_name }
      flavor: { get_param: flavor_small  }
      server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "svc"] ] }
      volume_size: { get_param: volume_large }
      image_id: { get_param: image_id }
      net: { get_param: net }
      share_net: { get_param: share_net }
      external_net: { get_param: external_net }
      assign_public_ip: true
      server_group: { get_resource: default_server_group }
  web:
    condition: create_web
    type: server_with_volume.yaml
    properties:
      key_name: { get_param: key_name }
      flavor: { get_param: flavor_medium  }
      server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "web"] ] }
      volume_size: { get_param: volume_large }
      image_id: { get_param: image_id }
      net: { get_param: net }
      share_net: { get_param: share_net }
      external_net: { get_param: external_net }
      assign_public_ip: true
      server_group: { get_resource: default_server_group }
  db:
    condition: create_db
    type: server_with_volume.yaml
    properties:
      key_name: { get_param: key_name }
      flavor: { get_param: flavor_small }
      server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "db"] ] }
      volume_size: { get_param: volume_small }
      extra_volume_size: { get_param: db_volume_size }
      extra_volume_type: { get_param: fast_volume_type }
      image_id: { get_param: image_id }
      net: { get_param: net }
      #share_net: { get_param: share_net }
      server_group: { get_resource: default_server_group }
  cluster_control:
    condition: create_cluster_control
    type: server_with_volume.yaml
    properties:
      key_name: { get_param: key_name }
      flavor: { get_param: flavor_small }
      server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "cluster_control"] ] }
      volume_size: { get_param: volume_small }
      image_id: { get_param: image_id }
      net: { get_param: net }
      #share_net: { get_param: share_net }
      server_group: { get_resource: default_server_group }
  frontend_db_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_frontend_db }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_database  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "frontend_db", "%index%" ]] }
          volume_size: { get_param: volume_small }
          extra_volume_size: { get_param: frontend_db_volume_size }
          extra_volume_type: { get_param: fast_volume_type }
          image_id: { get_param: image_id }
          net: { get_param: net }
          #share_net: { get_param: share_net }
          server_group: { get_resource: default_server_group }
  db_server_group: # not sure why, but putting frontend and backend db in same server group fails :(
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "db"] ] }
      policies: ["anti-affinity"]
  backend_db_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_backend_db }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_database  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "backend_db", "%index%" ]] }
          volume_size: { get_param: volume_small }
          extra_volume_size: { get_param: backend_db_volume_size }
          extra_volume_type: { get_param: fast_volume_type }
          image_id: { get_param: image_id }
          net: { get_param: net }
          server_group: { get_resource: db_server_group }
          #share_net: { get_param: share_net }
  kafka_pub:
    condition: create_kafka_pub
    type: server_with_volume.yaml
    properties:
      key_name: { get_param: key_name }
      flavor: { get_param: flavor_small  }
      server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "kafka_pub"] ] }
      volume_size: { get_param: volume_small }
      extra_volume_size: { get_param: kafka_pub_size }
      extra_volume_type: { get_param: slow_volume_type } 
      image_id: { get_param: image_id }
      net: { get_param: net }
      #share_net: { get_param: share_net }
      server_group: { get_resource: default_server_group }
      assign_public_ip: true
  kafka_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "kafka"] ] }
      policies: ["anti-affinity"]
  kafka_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_kafka }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_kafka  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "kafka", "%index%" ]] }
          volume_size: { get_param: volume_small }
          image_id: { get_param: image_id }
          net: { get_param: net }
          #share_net: { get_param: share_net }
          server_group: { get_resource: kafka_server_group }
  ingest_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "ingest"] ] }
      policies: ["anti-affinity"]
  ingest_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_ingest }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_small  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "ingest", "%index%" ]] }
          volume_size: { get_param: volume_small }
          image_id: { get_param: image_id }
          net: { get_param: net }
          share_net: { get_param: share_net }
          server_group: { get_resource: ingest_server_group }
  sherlock_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "sherlock"] ] }
      policies: ["anti-affinity"]
  sherlock_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_sherlock }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_xlarge  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "sherlock", "%index%" ]] }
          volume_size: { get_param: volume_small }
          extra_volume_size: { get_param: sherlock_volume_size }
          extra_volume_type: { get_param: fast_volume_type }
          image_id: { get_param: image_id }
          net: { get_param: net }
          share_net: { get_param: share_net }
          server_group: { get_resource: sherlock_server_group }
  filter_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "filter"] ] }
      policies: ["anti-affinity"]
  filter_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_filter }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_small  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "filter", "%index%" ]] }
          volume_size: { get_param: volume_small }
          image_id: { get_param: image_id }
          net: { get_param: net }
          share_net: { get_param: share_net }
          server_group: { get_resource: filter_server_group }
  cassandra_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "cassandra"] ] }
      policies: ["soft-anti-affinity"]
      #rules: { "max_server_per_host": 2 }
  cassandra_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_cassandra }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_large  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "cassandranodes", "%index%" ]] }
          volume_size: { get_param: volume_small }
          extra_volume_size: { get_param: cassandra_volume_size }
          extra_volume_type: { get_param: fast_volume_type }
          image_id: { get_param: image_id }
          net: { get_param: net }
          server_group: { get_resource: cassandra_server_group }
          #share_net: { get_param: share_net }
  parallel_ssh_server_group:
    type: OS::Nova::ServerGroup
    properties:
      name: { list_join: [ '-', [ { get_param: OS::stack_name }, "parallel_ssh"] ] }
      policies: ["soft-anti-affinity"]
      #rules: { "max_server_per_host": 2 }
  parallel_ssh_group:
    type: OS::Heat::ResourceGroup
    properties:
      count: { get_param: n_parallel_ssh }
      resource_def:
        type: server_with_volume.yaml
        properties:
          key_name: { get_param: key_name }
          flavor: { get_param: flavor_parallel_ssh  }
          server_name: { list_join: [ '-', [ { get_param: OS::stack_name }, "parallel_ssh", "%index%" ]] }
          volume_size: { get_param: volume_small }
          #extra_volume_size: { get_param:  }
          #extra_volume_type: { get_param:  }
          image_id: { get_param: image_id }
          net: { get_param: net }
          server_group: { get_resource: parallel_ssh_server_group }
          share_net: { get_param: share_net }

outputs:
  share_fs:
    description: Export locations
    value: { get_attr: [share_fs, export_locations] }
  svc:
    condition: create_svc
    description: Private IP
    value: { get_attr: [svc, first_address] }
  web:
    condition: create_web
    description: Private IP
    value: { get_attr: [web, first_address] }
  db:
    condition: create_db
    description: Private IP
    value: { get_attr: [db, first_address] }
  kafka_pub:
    condition: create_kafka_pub
    description: Private IP
    value: { get_attr: [kafka_pub, first_address] }
  kafka:
    description: Private IP list
    value: { get_attr: [kafka_group, first_address] }
  ingest:
    description: Private IP list
    value: { get_attr: [ingest_group, first_address] }
  sherlock:
    description: Private IP list
    value: { get_attr: [sherlock_group, first_address] }
  filter:
    description: Private IP list
    value: { get_attr: [filter_group, first_address] }
  cassandranodes:
    description: Private IP list
    value: { get_attr: [cassandra_group, first_address] }
  parallel_ssh:
    description: Private IP list
    value: { get_attr: [parallel_ssh_group, first_address] }
  frontend_db:
    description: Private IP list
    value: { get_attr: [frontend_db_group, first_address] }
  backend_db:
    description: Private IP list
    value: { get_attr: [backend_db_group, first_address] }
  cluster_control:
    condition: create_cluster_control
    description: Private IP
    value: { get_attr: [cluster_control, first_address] }