From 9b6fa349452d58157b1db5aac2c96cb906a03f9c Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 25 Aug 2015 13:56:59 -0400 Subject: [PATCH 01/15] Introducing docker with ability to run postgis, redis, and the test suite --- Dockerfile | 5 +++++ docker-compose.yml | 20 ++++++++++++++++++++ lib/database_helper.rb | 22 ++++++++++++++++++++++ lib/tasks/database.rake | 17 +++++++++++++++++ 4 files changed, 64 insertions(+) create mode 100644 Dockerfile create mode 100644 docker-compose.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2c62eb7 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM ruby:2.2.2-onbuild + +RUN apt-get install curl +RUN curl --silent --location https://deb.nodesource.com/setup_0.12 | bash - +RUN apt-get install --yes nodejs diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..801e8d7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,20 @@ +db: + image: mdillon/postgis +redis: + image: redis:3.0.3 +web: + image: citygram:latest + env_file: .env + environment: + DB_NAME: citygram_development + PG_USER: postgres + PG_HOST: citygram_db_1 + DATABASE_URL: postgres://postgres@citygram_db_1/citygram_development + command: bundle exec rackup -b '0.0.0.0' + volumes: + - .:/usr/src/app/citygram + ports: + - "9292:9292" + links: + - db + - redis diff --git a/lib/database_helper.rb b/lib/database_helper.rb index 24fa147..dc7dc19 100644 --- a/lib/database_helper.rb +++ b/lib/database_helper.rb @@ -79,7 +79,29 @@ def self.drop_db database.disconnect pg_command("dropdb #{db_name}") end + + def self.docker_create_db + pg_connection.exec("create database #{db_name}") + end + def self.docker_drop_db + database.disconnect + pg_connection.exec("drop database #{db_name}") + end + + def self.docker_reset + docker_drop_db + docker_create_db + migrate_db + end + + def self.pg_connection + PG.connect( + host: ENV["PG_HOST"], + user: ENV["PG_USER"] + ) + end + def self.schema_dump `rm #{schema_path}` pg_command("pg_dump -i -s -x -O -f #{schema_path} #{db_name}") diff --git a/lib/tasks/database.rake b/lib/tasks/database.rake index 48d1c4a..363ca68 100644 --- a/lib/tasks/database.rake +++ b/lib/tasks/database.rake @@ -35,6 +35,23 @@ namespace :db do task :schema_dump do Citygram::DatabaseHelper.schema_dump end + + namespace :docker do + desc 'Reset the database' + task :reset do + Citygram::DatabaseHelper.docker_reset + end + + desc 'Drop the database' + task :drop do + Citygram::DatabaseHelper.docker_drop_db + end + + desc 'Create the database' + task :create do + Citygram::DatabaseHelper.docker_create_db + end + end end namespace :g do From c913e0cba3cec45c6da39339ec291ec50e67f92e Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 25 Aug 2015 14:01:45 -0400 Subject: [PATCH 02/15] No longer rely on env file copying --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 801e8d7..2dec73f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,7 +4,7 @@ redis: image: redis:3.0.3 web: image: citygram:latest - env_file: .env + env_file: .env.sample environment: DB_NAME: citygram_development PG_USER: postgres From dd0c88a7aa2c460a4bfa83b8b2732ef928a0c504 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 25 Aug 2015 14:46:45 -0400 Subject: [PATCH 03/15] Adding setup rake task and docker test runner --- bin/test | 2 ++ lib/tasks/docker.rake | 8 ++++++++ 2 files changed, 10 insertions(+) create mode 100755 bin/test create mode 100644 lib/tasks/docker.rake diff --git a/bin/test b/bin/test new file mode 100755 index 0000000..d5fd20c --- /dev/null +++ b/bin/test @@ -0,0 +1,2 @@ +bundle exec rake docker:setup +docker-compose run web bundle exec rake test diff --git a/lib/tasks/docker.rake b/lib/tasks/docker.rake new file mode 100644 index 0000000..33cca7d --- /dev/null +++ b/lib/tasks/docker.rake @@ -0,0 +1,8 @@ +namespace :docker do + desc "Set up all containers needed for system operation" + task :setup do + system("docker create redis:3.0.3") + system("docker create mdillon/postgis") + system("docker create -t citygram .") + end +end From d3c2ec3a59ee8d62417ce9cf5ff909e84cab0677 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 25 Aug 2015 16:34:22 -0400 Subject: [PATCH 04/15] Begins with an empty docker installation and completes with a test run --- Dockerfile | 15 ++++++++++++++- bin/setup | 2 ++ bin/spec | 1 + bin/test | 2 -- lib/tasks/docker.rake | 2 +- 5 files changed, 18 insertions(+), 4 deletions(-) create mode 100755 bin/setup create mode 100755 bin/spec delete mode 100755 bin/test diff --git a/Dockerfile b/Dockerfile index 2c62eb7..d2800da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,18 @@ -FROM ruby:2.2.2-onbuild +FROM ruby:2.2.2 +# ExecJS runtime -- not optimal RUN apt-get install curl RUN curl --silent --location https://deb.nodesource.com/setup_0.12 | bash - RUN apt-get install --yes nodejs + +RUN mkdir -p /usr/src/app/citygram +WORKDIR /usr/src/app/citygram + +COPY Gemfile /usr/src/app/citygram/ +COPY Gemfile.lock /usr/src/app/citygram/ + +RUN /usr/local/bundle/bin/bundle install + +WORKDIR /usr/src/app/citygram + +# docker-compose will mount the working directory as the volume during that pass \ No newline at end of file diff --git a/bin/setup b/bin/setup new file mode 100755 index 0000000..1035917 --- /dev/null +++ b/bin/setup @@ -0,0 +1,2 @@ +bundle exec rake docker:setup +docker-compose run web bundle exec rake db:docker:create db:migrate diff --git a/bin/spec b/bin/spec new file mode 100755 index 0000000..d4e8529 --- /dev/null +++ b/bin/spec @@ -0,0 +1 @@ +docker-compose run web bundle exec rake test \ No newline at end of file diff --git a/bin/test b/bin/test deleted file mode 100755 index d5fd20c..0000000 --- a/bin/test +++ /dev/null @@ -1,2 +0,0 @@ -bundle exec rake docker:setup -docker-compose run web bundle exec rake test diff --git a/lib/tasks/docker.rake b/lib/tasks/docker.rake index 33cca7d..0e9380c 100644 --- a/lib/tasks/docker.rake +++ b/lib/tasks/docker.rake @@ -3,6 +3,6 @@ namespace :docker do task :setup do system("docker create redis:3.0.3") system("docker create mdillon/postgis") - system("docker create -t citygram .") + system("docker build -t citygram .") end end From ab80b8e171ab09ed58b56b903cdf8db0ba26d40f Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 25 Aug 2015 16:38:22 -0400 Subject: [PATCH 05/15] Reorganizing README to indicate Docker support mechanisms --- README.md | 55 ++++++++++++++++++++++++++++++++++++++++++------------- 1 file changed, 42 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index a152691..2adbd11 100644 --- a/README.md +++ b/README.md @@ -30,16 +30,17 @@ Citygram is a web application written in Ruby. * Job Queue: [Redis](http://redis.io/), [Sidekiq](https://github.com/mperham/sidekiq) * Tests: [RSpec](https://github.com/rspec), [FactoryGirl](https://github.com/thoughtbot/factory_girl), [Rack::Test](https://github.com/brynary/rack-test) -### Setup + +### Developing + +#### Standard Setup * Install Redis - `brew install redis` * [Install PostgreSQL](https://github.com/codeforamerica/howto/blob/master/PostgreSQL.md) * Install PostGIS -- refer to [these troubles](https://github.com/codeforamerica/citygram/issues/188) on Mac OS X * [Install Ruby](https://github.com/codeforamerica/howto/blob/master/Ruby.md) -In the command line, run the following: - -#### Install Dependencies +##### Install Dependencies ``` git clone https://github.com/codeforamerica/citygram.git @@ -50,7 +51,7 @@ gem install bundler bundle install ``` -#### Configure Environment +##### Configure Environment ``` cp .env.sample .env @@ -58,7 +59,41 @@ rake db:create db:migrate rake db:create db:migrate DATABASE_URL=postgres://localhost/citygram_test ``` -### Developing +##### Running tests + +Run all tests in the `spec/` directory. + +``` +rake +``` + + + +#### Docker Setup (experimental) + +Citygram is experimenting with a Docker-based approach to development to see if it aids in developer onboarding. + +There is no requirement other than Docker and a command-line. + +``` +bin/setup +bin/test +``` + +The first command will download and build all images needed to run Citygram in a local container. +The second command runs tests. You can make changes and run tests from there. + +##### Docker Migrations + +If your changes need migrations, this command will run them: + +``` +docker-compose run web bundle exec rake db:migrate +``` + +In fact, most arbitrary commands will work inside the container if you preface them with ```docker-compose run web``` + +### Running To boot up the complete application and run background jobs in development: @@ -67,6 +102,7 @@ bundle exec foreman start open http://localhost:5000/ ``` + ##### Single City Installation If you only need to support a single city you can specify the ROOT_CITY_TAG to bypass the index and load one city. @@ -95,10 +131,3 @@ rake digests:send_if_digest_day [![Heroku Scheduler](https://cloud.githubusercontent.com/assets/81055/8840908/732942c2-30b5-11e5-8af7-06b9e169d281.png)](https://devcenter.heroku.com/articles/scheduler) -### Testing - -Run all tests in the `spec/` directory. - -``` -rake -``` From 6a0fb3ae4a1f777fb941eb31b65016ca1e4006eb Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 27 Oct 2015 18:09:40 -0400 Subject: [PATCH 06/15] Moved to Ruby 2.2.3 --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index d2800da..e028f56 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM ruby:2.2.2 +FROM ruby:2.2.3 # ExecJS runtime -- not optimal -RUN apt-get install curl +RUN apt-get update && apt-get install curl RUN curl --silent --location https://deb.nodesource.com/setup_0.12 | bash - RUN apt-get install --yes nodejs From fb131826fb57541a8b379d04b47a70f3dac69e26 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 27 Oct 2015 18:09:56 -0400 Subject: [PATCH 07/15] Depending on bundle is counter-productive --- bin/setup | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/setup b/bin/setup index 1035917..62da596 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1 @@ -bundle exec rake docker:setup -docker-compose run web bundle exec rake db:docker:create db:migrate +docker-compose run citygram rake db:docker:create db:migrate From 9b727f27913c49367e944bb2492a19da3ebb627f Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 27 Oct 2015 18:10:47 -0400 Subject: [PATCH 08/15] Additional prep for before publishing to Docker Hub --- bin/setup | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/setup b/bin/setup index 62da596..a3e475e 100755 --- a/bin/setup +++ b/bin/setup @@ -1 +1,2 @@ +docker build -t citygram:latest . docker-compose run citygram rake db:docker:create db:migrate From 32cef86140a87cf09c4d4fb69517c83305ad6219 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Tue, 27 Oct 2015 18:11:07 -0400 Subject: [PATCH 09/15] Can run tests successfully --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2dec73f..0a3404c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ db: image: mdillon/postgis redis: image: redis:3.0.3 -web: +citygram: image: citygram:latest env_file: .env.sample environment: @@ -10,7 +10,7 @@ web: PG_USER: postgres PG_HOST: citygram_db_1 DATABASE_URL: postgres://postgres@citygram_db_1/citygram_development - command: bundle exec rackup -b '0.0.0.0' + command: bundle exec foreman start volumes: - .:/usr/src/app/citygram ports: From 5b8125a98729d2777745667a3c361eac99d4278b Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 28 Oct 2015 09:06:33 -0400 Subject: [PATCH 10/15] Updates to composition, idempotent DB creation --- bin/spec | 2 +- lib/database_helper.rb | 25 ++++++++++++++++++++----- 2 files changed, 21 insertions(+), 6 deletions(-) diff --git a/bin/spec b/bin/spec index d4e8529..f2a6447 100755 --- a/bin/spec +++ b/bin/spec @@ -1 +1 @@ -docker-compose run web bundle exec rake test \ No newline at end of file +docker-compose run citygram rake test \ No newline at end of file diff --git a/lib/database_helper.rb b/lib/database_helper.rb index dc7dc19..c23ebef 100644 --- a/lib/database_helper.rb +++ b/lib/database_helper.rb @@ -79,29 +79,44 @@ def self.drop_db database.disconnect pg_command("dropdb #{db_name}") end - + def self.docker_create_db - pg_connection.exec("create database #{db_name}") + pg_connection.exec(<<-SQL + DO + $do$ + BEGIN + + IF EXISTS (SELECT 1 FROM pg_database WHERE datname = '#{db_name}') THEN + RAISE NOTICE 'Citygram database exists, you are ready to test.'; + ELSE + PERFORM dblink_exec('dbname=' || current_database() -- current db + , 'CREATE DATABASE #{db_name}'); + END IF; + + END + $do$ +SQL + ) end def self.docker_drop_db database.disconnect pg_connection.exec("drop database #{db_name}") end - + def self.docker_reset docker_drop_db docker_create_db migrate_db end - + def self.pg_connection PG.connect( host: ENV["PG_HOST"], user: ENV["PG_USER"] ) end - + def self.schema_dump `rm #{schema_path}` pg_command("pg_dump -i -s -x -O -f #{schema_path} #{db_name}") From 6c425a0b550aac23024765b8a788c9caf239907a Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 28 Oct 2015 11:24:40 -0400 Subject: [PATCH 11/15] Closer to running application --- Procfile.docker | 2 ++ bin/citygram | 1 + docker-compose.yml | 8 +++++--- 3 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 Procfile.docker create mode 100755 bin/citygram diff --git a/Procfile.docker b/Procfile.docker new file mode 100644 index 0000000..b3fc609 --- /dev/null +++ b/Procfile.docker @@ -0,0 +1,2 @@ +web: bundle exec unicorn -c ./config/unicorn.rb +worker: bundle exec sidekiq -c 5 -r ./app.rb diff --git a/bin/citygram b/bin/citygram new file mode 100755 index 0000000..c2da170 --- /dev/null +++ b/bin/citygram @@ -0,0 +1 @@ +docker-compose run citygram diff --git a/docker-compose.yml b/docker-compose.yml index 0a3404c..9a1d9a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,18 +3,20 @@ db: redis: image: redis:3.0.3 citygram: - image: citygram:latest + build: . env_file: .env.sample environment: DB_NAME: citygram_development PG_USER: postgres PG_HOST: citygram_db_1 DATABASE_URL: postgres://postgres@citygram_db_1/citygram_development - command: bundle exec foreman start + REDIS_PROVIDER: REDIS_URL + REDIS_URL: redis://redis:6379/ + command: bundle exec foreman start -f Procfile.docker volumes: - .:/usr/src/app/citygram ports: - - "9292:9292" + - "5000:5000" links: - db - redis From 0cb1f3226f8ee29ae4817dfac8d845d3f77e18d3 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 28 Oct 2015 11:49:44 -0400 Subject: [PATCH 12/15] Available via docker-machine --- bin/citygram | 2 +- docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/citygram b/bin/citygram index c2da170..0fb21a2 100755 --- a/bin/citygram +++ b/bin/citygram @@ -1 +1 @@ -docker-compose run citygram +docker-compose run --service-ports citygram diff --git a/docker-compose.yml b/docker-compose.yml index 9a1d9a9..a35fa61 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,11 +12,11 @@ citygram: DATABASE_URL: postgres://postgres@citygram_db_1/citygram_development REDIS_PROVIDER: REDIS_URL REDIS_URL: redis://redis:6379/ - command: bundle exec foreman start -f Procfile.docker + command: bundle exec rackup -o 0.0.0.0 -p 9292 volumes: - .:/usr/src/app/citygram ports: - - "5000:5000" + - "9292:9292" links: - db - redis From 5a58f74293f8ba3e8483cd3a55cc6feb38385af8 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 28 Oct 2015 11:57:42 -0400 Subject: [PATCH 13/15] Adding new migrations to publisher download --- lib/tasks/publishers.rake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tasks/publishers.rake b/lib/tasks/publishers.rake index 8f0ec1a..1c34305 100644 --- a/lib/tasks/publishers.rake +++ b/lib/tasks/publishers.rake @@ -11,8 +11,9 @@ namespace :publishers do pub_file = open("https://www.citygram.org/publishers.json").read publishers = JSON.parse(pub_file) Citygram::Models::Publisher.set_allowed_columns( - :title, :endpoint, :active, :visible, - :city, :state, :icon, :description, :tags + :title, :endpoint, :active, :visible, + :city, :state, :icon, :description, :tags, + :event_display_endpoint, :events_are_polygons ) publishers.each do |pub| pub.delete("id") From 59e431896808bd56c6c3dcd9462cdcd07ba00447 Mon Sep 17 00:00:00 2001 From: Jim Van Fleet Date: Wed, 28 Oct 2015 13:11:04 -0400 Subject: [PATCH 14/15] Setup script can run from scratch, and can be re-run safely --- bin/setup | 5 +++-- lib/database_helper.rb | 15 +++++++++------ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/bin/setup b/bin/setup index a3e475e..3a3db65 100755 --- a/bin/setup +++ b/bin/setup @@ -1,2 +1,3 @@ -docker build -t citygram:latest . -docker-compose run citygram rake db:docker:create db:migrate +docker-compose build +docker-compose run citygram rake db:docker:create +docker-compose run citygram rake db:migrate \ No newline at end of file diff --git a/lib/database_helper.rb b/lib/database_helper.rb index c23ebef..607d6a8 100644 --- a/lib/database_helper.rb +++ b/lib/database_helper.rb @@ -81,22 +81,25 @@ def self.drop_db end def self.docker_create_db - pg_connection.exec(<<-SQL + con = pg_connection + @exists = false + con.set_notice_receiver{|result| @exists = true } + val = con.exec(<<-SQL DO $do$ BEGIN IF EXISTS (SELECT 1 FROM pg_database WHERE datname = '#{db_name}') THEN RAISE NOTICE 'Citygram database exists, you are ready to test.'; - ELSE - PERFORM dblink_exec('dbname=' || current_database() -- current db - , 'CREATE DATABASE #{db_name}'); END IF; - END $do$ SQL ) + if !@exists + puts "Creating DB" + pg_connection.exec("create database #{db_name}") + end end def self.docker_drop_db @@ -118,7 +121,7 @@ def self.pg_connection end def self.schema_dump - `rm #{schema_path}` + `rm #{schema_path}` if File.exist?(schema_path) pg_command("pg_dump -i -s -x -O -f #{schema_path} #{db_name}") end From 305de1dc50e595c42399355cbc4e72c1090101f4 Mon Sep 17 00:00:00 2001 From: Jon Roes Date: Fri, 12 Feb 2016 15:13:57 -0500 Subject: [PATCH 15/15] implement bin/test, fix Dockerfile apt-get line, update docs --- Dockerfile | 4 ++-- README.md | 6 ++---- bin/test | 1 + 3 files changed, 5 insertions(+), 6 deletions(-) create mode 100755 bin/test diff --git a/Dockerfile b/Dockerfile index e028f56..ec5a3e0 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ FROM ruby:2.2.3 # ExecJS runtime -- not optimal -RUN apt-get update && apt-get install curl +RUN apt-get update && apt-get install curl --yes RUN curl --silent --location https://deb.nodesource.com/setup_0.12 | bash - RUN apt-get install --yes nodejs @@ -15,4 +15,4 @@ RUN /usr/local/bundle/bin/bundle install WORKDIR /usr/src/app/citygram -# docker-compose will mount the working directory as the volume during that pass \ No newline at end of file +# docker-compose will mount the working directory as the volume during that pass diff --git a/README.md b/README.md index da28abb..8207a72 100644 --- a/README.md +++ b/README.md @@ -155,9 +155,7 @@ The second command runs tests. You can make changes and run tests from there. If your changes need migrations, this command will run them: ``` -docker-compose run web bundle exec rake db:migrate +docker-compose run citygram bundle exec rake db:migrate ``` -In fact, most arbitrary commands will work inside the container if you preface them with ```docker-compose run web``` - - +In fact, most arbitrary commands will work inside the container if you preface them with ```docker-compose run citygram``` diff --git a/bin/test b/bin/test new file mode 100755 index 0000000..73473e9 --- /dev/null +++ b/bin/test @@ -0,0 +1 @@ +docker-compose run citygram bundle exec rake