diff --git a/.gitignore b/.gitignore index 10afca85..026d9e89 100644 --- a/.gitignore +++ b/.gitignore @@ -15,5 +15,8 @@ !/log/.keep !/tmp/.keep +# Ignore coverage data +/coverage + # Ignore Byebug command history file. .byebug_history diff --git a/Gemfile b/Gemfile index 29ee58d1..f6fe5e65 100644 --- a/Gemfile +++ b/Gemfile @@ -1,14 +1,15 @@ source 'https://rubygems.org' +ruby '2.4.1' # Bundle edge Rails instead: gem 'rails', github: 'rails/rails' -gem 'rails', '~> 5.0.0', '>= 5.0.0.1' +gem 'rails', '~> 5.0.2' # Use postgresql as the database for Active Record -gem 'pg', '~> 0.18' +gem 'pg', '~> 0.20' gem 'activerecord-postgis-adapter' # Build JSON APIs with ease. Read more: https://github.com/rails/jbuilder -gem 'jbuilder', '~> 2.5' +gem 'jbuilder', '~> 2.6' # Use Redis adapter to run Action Cable in production # gem 'redis', '~> 3.0' # Use ActiveModel has_secure_password @@ -20,24 +21,35 @@ gem 'faraday' gem 'clockwork' gem 'sidekiq' gem 'sidekiq-symbols' +gem 'foreman' gem 'typhoeus' gem 'nokogiri' -gem 'ox' +gem 'ox', '2.4.12' +gem 'rubyzip' + +gem 'puma' +gem 'newrelic_rpm' # Use Capistrano for deployment # gem 'capistrano-rails', group: :development group :development, :test do # Call 'byebug' anywhere in the code to stop execution and get a debugger console - gem 'byebug', platform: :mri + gem 'byebug', platforms: [:mri, :mingw, :x64_mingw] gem 'rspec-rails' end +group :test do + gem 'simplecov' + gem 'codeclimate-test-reporter', '~> 1.0.8' + gem 'rspec_junit_formatter' +end + group :development do # Access an IRB console on exception pages or by using <%= console %> anywhere in the code. - gem 'web-console' - gem 'listen', '~> 3.0.5' + gem 'web-console', '>= 3.3.0' + gem 'listen', '>= 3.0.5', '< 3.2' # Spring speeds up development by keeping your application running in the background. Read more: https://github.com/rails/spring gem 'spring' gem 'spring-watcher-listen', '~> 2.0.0' diff --git a/Gemfile.lock b/Gemfile.lock index 1bf8a2a9..b553e6dc 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,171 +1,193 @@ GEM remote: https://rubygems.org/ specs: - actioncable (5.0.0.1) - actionpack (= 5.0.0.1) - nio4r (~> 1.2) + actioncable (5.0.2) + actionpack (= 5.0.2) + nio4r (>= 1.2, < 3.0) websocket-driver (~> 0.6.1) - actionmailer (5.0.0.1) - actionpack (= 5.0.0.1) - actionview (= 5.0.0.1) - activejob (= 5.0.0.1) + actionmailer (5.0.2) + actionpack (= 5.0.2) + actionview (= 5.0.2) + activejob (= 5.0.2) mail (~> 2.5, >= 2.5.4) rails-dom-testing (~> 2.0) - actionpack (5.0.0.1) - actionview (= 5.0.0.1) - activesupport (= 5.0.0.1) + actionpack (5.0.2) + actionview (= 5.0.2) + activesupport (= 5.0.2) rack (~> 2.0) rack-test (~> 0.6.3) rails-dom-testing (~> 2.0) rails-html-sanitizer (~> 1.0, >= 1.0.2) - actionview (5.0.0.1) - activesupport (= 5.0.0.1) + actionview (5.0.2) + activesupport (= 5.0.2) builder (~> 3.1) erubis (~> 2.7.0) rails-dom-testing (~> 2.0) - rails-html-sanitizer (~> 1.0, >= 1.0.2) - activejob (5.0.0.1) - activesupport (= 5.0.0.1) + rails-html-sanitizer (~> 1.0, >= 1.0.3) + activejob (5.0.2) + activesupport (= 5.0.2) globalid (>= 0.3.6) - activemodel (5.0.0.1) - activesupport (= 5.0.0.1) - activerecord (5.0.0.1) - activemodel (= 5.0.0.1) - activesupport (= 5.0.0.1) + activemodel (5.0.2) + activesupport (= 5.0.2) + activerecord (5.0.2) + activemodel (= 5.0.2) + activesupport (= 5.0.2) arel (~> 7.0) - activerecord-postgis-adapter (4.0.0) + activerecord-postgis-adapter (4.0.3) activerecord (~> 5.0.0) rgeo-activerecord (~> 5.0.0) - activesupport (5.0.0.1) + activesupport (5.0.2) concurrent-ruby (~> 1.0, >= 1.0.2) i18n (~> 0.7) minitest (~> 5.1) tzinfo (~> 1.1) - arel (7.1.2) - builder (3.2.2) + arel (7.1.4) + bindex (0.5.0) + builder (3.2.3) byebug (9.0.6) - clockwork (2.0.0) + clockwork (2.0.2) activesupport tzinfo - concurrent-ruby (1.0.2) - connection_pool (2.2.0) - debug_inspector (0.0.2) - diff-lcs (1.2.5) - dotenv (2.1.1) + codeclimate-test-reporter (1.0.8) + simplecov (<= 0.13) + concurrent-ruby (1.0.5) + connection_pool (2.2.1) + diff-lcs (1.3) + docile (1.1.5) + dotenv (2.2.1) erubis (2.7.0) - ethon (0.9.0) + ethon (0.10.1) ffi (>= 1.3.0) - faraday (0.9.2) + faraday (0.12.1) multipart-post (>= 1.2, < 3) - ffi (1.9.14) - globalid (0.3.7) - activesupport (>= 4.1.0) - i18n (0.7.0) - jbuilder (2.6.0) - activesupport (>= 3.0.0, < 5.1) + ffi (1.9.18) + foreman (0.84.0) + thor (~> 0.19.1) + globalid (0.4.0) + activesupport (>= 4.2.0) + i18n (0.8.1) + jbuilder (2.6.3) + activesupport (>= 3.0.0, < 5.2) multi_json (~> 1.2) - listen (3.0.8) + json (2.1.0) + listen (3.1.5) rb-fsevent (~> 0.9, >= 0.9.4) rb-inotify (~> 0.9, >= 0.9.7) + ruby_dep (~> 1.2) loofah (2.0.3) nokogiri (>= 1.5.9) - mail (2.6.4) + mail (2.6.5) mime-types (>= 1.16, < 4) method_source (0.8.2) mime-types (3.1) mime-types-data (~> 3.2015) mime-types-data (3.2016.0521) mini_portile2 (2.1.0) - minitest (5.9.1) + minitest (5.10.1) multi_json (1.12.1) multipart-post (2.0.0) - nio4r (1.2.1) - nokogiri (1.6.8.1) + newrelic_rpm (4.1.0.333) + nio4r (2.0.0) + nokogiri (1.7.1) mini_portile2 (~> 2.1.0) - ox (2.4.4) - pg (0.19.0) + ox (2.4.12) + pg (0.20.0) + puma (3.8.2) rack (2.0.1) + rack-protection (1.5.3) + rack rack-test (0.6.3) rack (>= 1.0) - rails (5.0.0.1) - actioncable (= 5.0.0.1) - actionmailer (= 5.0.0.1) - actionpack (= 5.0.0.1) - actionview (= 5.0.0.1) - activejob (= 5.0.0.1) - activemodel (= 5.0.0.1) - activerecord (= 5.0.0.1) - activesupport (= 5.0.0.1) + rails (5.0.2) + actioncable (= 5.0.2) + actionmailer (= 5.0.2) + actionpack (= 5.0.2) + actionview (= 5.0.2) + activejob (= 5.0.2) + activemodel (= 5.0.2) + activerecord (= 5.0.2) + activesupport (= 5.0.2) bundler (>= 1.3.0, < 2.0) - railties (= 5.0.0.1) + railties (= 5.0.2) sprockets-rails (>= 2.0.0) - rails-dom-testing (2.0.1) + rails-dom-testing (2.0.2) activesupport (>= 4.2.0, < 6.0) - nokogiri (~> 1.6.0) + nokogiri (~> 1.6) rails-html-sanitizer (1.0.3) loofah (~> 2.0) - railties (5.0.0.1) - actionpack (= 5.0.0.1) - activesupport (= 5.0.0.1) + railties (5.0.2) + actionpack (= 5.0.2) + activesupport (= 5.0.2) method_source rake (>= 0.8.7) thor (>= 0.18.1, < 2.0) - rake (11.3.0) - rb-fsevent (0.9.7) - rb-inotify (0.9.7) + rake (12.0.0) + rb-fsevent (0.9.8) + rb-inotify (0.9.8) ffi (>= 0.5.0) - redis (3.3.0) - rgeo (0.5.3) - rgeo-activerecord (5.0.0) + redis (3.3.3) + rgeo (0.6.0) + rgeo-activerecord (5.0.1) activerecord (~> 5.0.0) rgeo (~> 0.3) - rspec-core (3.5.2) - rspec-support (~> 3.5.0) - rspec-expectations (3.5.0) + rspec-core (3.6.0) + rspec-support (~> 3.6.0) + rspec-expectations (3.6.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.5.0) - rspec-mocks (3.5.0) + rspec-support (~> 3.6.0) + rspec-mocks (3.6.0) diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.5.0) - rspec-rails (3.5.1) + rspec-support (~> 3.6.0) + rspec-rails (3.6.0) actionpack (>= 3.0) activesupport (>= 3.0) railties (>= 3.0) - rspec-core (~> 3.5.0) - rspec-expectations (~> 3.5.0) - rspec-mocks (~> 3.5.0) - rspec-support (~> 3.5.0) - rspec-support (3.5.0) - sidekiq (4.1.1) + rspec-core (~> 3.6.0) + rspec-expectations (~> 3.6.0) + rspec-mocks (~> 3.6.0) + rspec-support (~> 3.6.0) + rspec-support (3.6.0) + rspec_junit_formatter (0.2.3) + builder (< 4) + rspec-core (>= 2, < 4, != 2.12.0) + ruby_dep (1.5.0) + rubyzip (1.2.1) + sidekiq (5.0.0) concurrent-ruby (~> 1.0) connection_pool (~> 2.2, >= 2.2.0) - redis (~> 3.2, >= 3.2.1) + rack-protection (>= 1.5.0) + redis (~> 3.3, >= 3.3.3) sidekiq-symbols (0.1.1) sidekiq - spring (2.0.0) + simplecov (0.13.0) + docile (~> 1.1.0) + json (>= 1.8, < 3) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + spring (2.0.1) activesupport (>= 4.2) spring-watcher-listen (2.0.1) listen (>= 2.7, < 4.0) spring (>= 1.2, < 3.0) - sprockets (3.7.0) + sprockets (3.7.1) concurrent-ruby (~> 1.0) rack (> 1, < 3) sprockets-rails (3.2.0) actionpack (>= 4.0) activesupport (>= 4.0) sprockets (>= 3.0.0) - thor (0.19.1) - thread_safe (0.3.5) - typhoeus (1.0.2) + thor (0.19.4) + thread_safe (0.3.6) + typhoeus (1.1.2) ethon (>= 0.9.0) - tzinfo (1.2.2) + tzinfo (1.2.3) thread_safe (~> 0.1) - web-console (3.3.1) + web-console (3.5.0) actionview (>= 5.0) activemodel (>= 5.0) - debug_inspector + bindex (>= 0.4.0) railties (>= 5.0) - websocket-driver (0.6.4) + websocket-driver (0.6.5) websocket-extensions (>= 0.1.0) websocket-extensions (0.1.2) @@ -176,22 +198,32 @@ DEPENDENCIES activerecord-postgis-adapter byebug clockwork + codeclimate-test-reporter (~> 1.0.8) dotenv faraday - jbuilder (~> 2.5) - listen (~> 3.0.5) + foreman + jbuilder (~> 2.6) + listen (>= 3.0.5, < 3.2) + newrelic_rpm nokogiri - ox - pg (~> 0.18) - rails (~> 5.0.0, >= 5.0.0.1) + ox (= 2.4.12) + pg (~> 0.20) + puma + rails (~> 5.0.2) rspec-rails + rspec_junit_formatter + rubyzip sidekiq sidekiq-symbols + simplecov spring spring-watcher-listen (~> 2.0.0) typhoeus tzinfo-data - web-console + web-console (>= 3.3.0) + +RUBY VERSION + ruby 2.4.1p111 BUNDLED WITH - 1.12.5 + 1.14.6 diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..1209eb9c --- /dev/null +++ b/LICENSE @@ -0,0 +1,297 @@ + European Union Public Licence + V. 1.1 + + EUPL (c) the European Community 2007 + + +This European Union Public Licence (the "EUPL") applies to the Work or Software +(as defined below) which is provided under the terms of this Licence. Any use +of the Work, other than as authorised under this Licence is prohibited (to the +extent such use is covered by a right of the copyright holder of the Work). + +The Original Work is provided under the terms of this Licence when the Licensor +(as defined below) has placed the following notice immediately following the +copyright notice for the Original Work: + + Licensed under the EUPL V.1.1 + +or has expressed by any other mean his willingness to license under the EUPL. + + + 1. Definitions + +In this Licence, the following terms have the following meaning: + + * The Licence: this Licence. + + * The Original Work or the Software: the software distributed and/or + communicated by the Licensor under this Licence, available as Source Code + and also as Executable Code as the case may be. + + * Derivative Works: the works or software that could be created by the + Licensee, based upon the Original Work or modifications thereof. This + Licence does not define the extent of modification or dependence on the + Original Work required in order to classify a work as a Derivative Work; + this extent is determined by copyright law applicable in the country + mentioned in Article 15. + + * The Work: the Original Work and/or its Derivative Works. + + * The Source Code: the human-readable form of the Work which is the most + convenient for people to study and modify. + + * The Executable Code: any code which has generally been compiled and which is + meant to be interpreted by a computer as a program. + + * The Licensor: the natural or legal person that distributes and/or + communicates the Work under the Licence. + + * Contributor(s): any natural or legal person who modifies the Work under the + Licence, or otherwise contributes to the creation of a Derivative Work. + + * The Licensee or "You": any natural or legal person who makes any usage of + the Software under the terms of the Licence. + + * Distribution and/or Communication: any act of selling, giving, lending, + renting, distributing, communicating, transmitting, or otherwise making + available, on-line or off-line, copies of the Work or providing access to + its essential functionalities at the disposal of any other natural or legal + person. + + + 2. Scope of the rights granted by the Licence + +The Licensor hereby grants You a world-wide, royalty-free, non-exclusive, +sublicensable licence to do the following, for the duration of copyright vested +in the Original Work: + + * use the Work in any circumstance and for all usage, + * reproduce the Work, + * modify the Original Work, and make Derivative Works based upon the Work, + * communicate to the public, including the right to make available or display + the Work or copies thereof to the public and perform publicly, as the case + may be, the Work, + * distribute the Work or copies thereof, + * lend and rent the Work or copies thereof, + * sub-license rights in the Work or copies thereof. + +Those rights can be exercised on any media, supports and formats, whether now +known or later invented, as far as the applicable law permits so. + +In the countries where moral rights apply, the Licensor waives his right to +exercise his moral right to the extent allowed by law in order to make +effective the licence of the economic rights here above listed. + +The Licensor grants to the Licensee royalty-free, non exclusive usage rights to +any patents held by the Licensor, to the extent necessary to make use of the +rights granted on the Work under this Licence. + + + 3. Communication of the Source Code + +The Licensor may provide the Work either in its Source Code form, or as +Executable Code. If the Work is provided as Executable Code, the Licensor +provides in addition a machine-readable copy of the Source Code of the Work +along with each copy of the Work that the Licensor distributes or indicates, in +a notice following the copyright notice attached to the Work, a repository +where the Source Code is easily and freely accessible for as long as the +Licensor continues to distribute and/or communicate the Work. + + + 4. Limitations on copyright + +Nothing in this Licence is intended to deprive the Licensee of the benefits +from any exception or limitation to the exclusive rights of the rights owners +in the Original Work or Software, of the exhaustion of those rights or of other +applicable limitations thereto. + + + 5. Obligations of the Licensee + +The grant of the rights mentioned above is subject to some restrictions and +obligations imposed on the Licensee. Those obligations are the following: + + - Attribution right: the Licensee shall keep intact all copyright, patent or + trademarks notices and all notices that refer to the Licence and to the + disclaimer of warranties. The Licensee must include a copy of such notices + and a copy of the Licence with every copy of the Work he/she distributes + and/or communicates. The Licensee must cause any Derivative Work to carry + prominent notices stating that the Work has been modified and the date of + modification. + + - Copyleft clause: If the Licensee distributes and/or communicates copies of + the Original Works or Derivative Works based upon the Original Work, this + Distribution and/or Communication will be done under the terms of this + Licence or of a later version of this Licence unless the Original Work is + expressly distributed only under this version of the Licence. The Licensee + (becoming Licensor) cannot offer or impose any additional terms or + conditions on the Work or Derivative Work that alter or restrict the terms + of the Licence. + + - Compatibility clause: If the Licensee Distributes and/or Communicates + Derivative Works or copies thereof based upon both the Original Work and + another work licensed under a Compatible Licence, this Distribution and/or + Communication can be done under the terms of this Compatible Licence. For + the sake of this clause, "Compatible Licence" refers to the licences listed + in the appendix attached to this Licence. Should the Licensee's obligations + under the Compatible Licence conflict with his/her obligations under this + Licence, the obligations of the Compatible Licence shall prevail. + + - Provision of Source Code: When distributing and/or communicating copies of + the Work, the Licensee will provide a machine-readable copy of the Source + Code or indicate a repository where this Source will be easily and freely + available for as long as the Licensee continues to distribute and/or + communicate the Work. Legal Protection: This Licence does not grant + permission to use the trade names, trademarks, service marks, or names of + the Licensor, except as required for reasonable and customary use in + describing the origin of the Work and reproducing the content of the + copyright notice. + + + 6. Chain of Authorship + +The original Licensor warrants that the copyright in the Original Work granted +hereunder is owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each Contributor warrants that the copyright in the modifications he/she brings +to the Work are owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each time You accept the Licence, the original Licensor and subsequent +Contributors grant You a licence to their contributions to the Work, under the +terms of this Licence. + + + 7. Disclaimer of Warranty + +The Work is a work in progress, which is continuously improved by numerous +contributors. It is not a finished work and may therefore contain defects or +"bugs" inherent to this type of software development. + +For the above reason, the Work is provided under the Licence on an "as is" +basis and without warranties of any kind concerning the Work, including without +limitation merchantability, fitness for a particular purpose, absence of +defects or errors, accuracy, non-infringement of intellectual property rights +other than copyright as stated in Article 6 of this Licence. + +This disclaimer of warranty is an essential part of the Licence and a condition +for the grant of any rights to the Work. + + + 8. Disclaimer of Liability + +Except in the cases of wilful misconduct or damages directly caused to natural +persons, the Licensor will in no event be liable for any direct or indirect, +material or moral, damages of any kind, arising out of the Licence or of the +use of the Work, including without limitation, damages for loss of goodwill, +work stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such +damage. However, the Licensor will be liable under statutory product liability +laws as far such laws apply to the Work. + + + 9. Additional agreements + +While distributing the Original Work or Derivative Works, You may choose to +conclude an additional agreement to offer, and charge a fee for, acceptance of +support, warranty, indemnity, or other liability obligations and/or services +consistent with this Licence. However, in accepting such obligations, You may +act only on your own behalf and on your sole responsibility, not on behalf of +the original Licensor or any other Contributor, and only if You agree to +indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against such Contributor by the fact You have +accepted any such warranty or additional liability. + + + 10. Acceptance of the Licence + +The provisions of this Licence can be accepted by clicking on an icon "I agree" +placed under the bottom of a window displaying the text of this Licence or by +affirming consent in any other similar way, in accordance with the rules of +applicable law. Clicking on that icon indicates your clear and irrevocable +acceptance of this Licence and all of its terms and conditions. + +Similarly, you irrevocably accept this Licence and all of its terms and +conditions by exercising any rights granted to You by Article 2 of this +Licence, such as the use of the Work, the creation by You of a Derivative Work +or the Distribution and/or Communication by You of the Work or copies thereof. + + + 11. Information to the public + +In case of any Distribution and/or Communication of the Work by means of +electronic communication by You (for example, by offering to download the Work +from a remote location) the distribution channel or media (for example, a +website) must at least provide to the public the information requested by the +applicable law regarding the Licensor, the Licence and the way it may be +accessible, concluded, stored and reproduced by the Licensee. + + + 12. Termination of the Licence + +The Licence and the rights granted hereunder will terminate automatically upon +any breach by the Licensee of the terms of the Licence. + +Such a termination will not terminate the licences of any person who has +received the Work from the Licensee under the Licence, provided such persons +remain in full compliance with the Licence. + + + 13. Miscellaneous + +Without prejudice of Article 9 above, the Licence represents the complete +agreement between the Parties as to the Work licensed hereunder. + +If any provision of the Licence is invalid or unenforceable under applicable +law, this will not affect the validity or enforceability of the Licence as a +whole. Such provision will be construed and/or reformed so as necessary to make +it valid and enforceable. + +The European Commission may publish other linguistic versions and/or new +versions of this Licence, so far this is required and reasonable, without +reducing the scope of the rights granted by the Licence. New versions of the +Licence will be published with a unique version number. + +All linguistic versions of this Licence, approved by the European Commission, +have identical value. Parties can take advantage of the linguistic version of +their choice. + + + 14. Jurisdiction + +Any litigation resulting from the interpretation of this License, arising +between the European Commission, as a Licensor, and any Licensee, will be +subject to the jurisdiction of the Court of Justice of the European +Communities, as laid down in article 238 of the Treaty establishing the +European Community. + +Any litigation arising between Parties, other than the European Commission, and +resulting from the interpretation of this License, will be subject to the +exclusive jurisdiction of the competent court where the Licensor resides or +conducts its primary business. + + + 15. Applicable Law + +This Licence shall be governed by the law of the European Union country where +the Licensor resides or has his registered office. + +This licence shall be governed by the Belgian law if: + + * a litigation arises between the European Commission, as a Licensor, and any + Licensee; + * the Licensor, other than the European Commission, has no residence or + registered office inside a European Union country. + + + Appendix + +"Compatible Licences" according to article 5 EUPL are: + + * GNU General Public License (GNU GPL) v. 2 + * Open Software License (OSL) v. 2.1, v. 3.0 + * Common Public License v. 1.0 + * Eclipse Public License v. 1.0 + * Cecill v. 2.0 + diff --git a/Procfile b/Procfile new file mode 100644 index 00000000..b928d182 --- /dev/null +++ b/Procfile @@ -0,0 +1,3 @@ +web: bundle exec puma -C config/puma.rb +worker: bundle exec sidekiq -C config/sidekiq.yml +clock: bundle exec clockwork clock.rb diff --git a/README.md b/README.md new file mode 100644 index 00000000..0aa89e5d --- /dev/null +++ b/README.md @@ -0,0 +1,13 @@ +# Harvester + +Harvester is an open-source application that downloads data from various public sources and pushes data to [ekosystem.slovensko.digital](https://ekosystem.slovensko.digital/). + +[![CircleCI](https://circleci.com/gh/slovensko-digital/harvester.ecosystem.svg?style=shield&circle-token=4ceebb8c3fa816ecc9ab67491f55f665ea9d8661)](https://circleci.com/gh/slovensko-digital/harvester.ecosystem) +[![Dependency Status](https://gemnasium.com/slovensko-digital/harvester.ecosystem.png)](https://gemnasium.com/slovensko-digital/harvester.ecosystem) +[![Code Climate](https://codeclimate.com/github/slovensko-digital/harvester.ecosystem.png)](https://codeclimate.com/github/slovensko-digital/harvester.ecosystem) +[![Test Coverage](https://codeclimate.com/github/slovensko-digital/harvester.ecosystem/badges/coverage.svg)](https://codeclimate.com/github/slovensko-digital/harvester.ecosystem/coverage) +[![Inch CI](https://inch-ci.org/github/slovensko-digital/harvester.ecosystem.svg)](https://inch-ci.org/github/slovensko-digital/harvester.ecosystem) + +## Contributing + +Feel free to contribute other datasets / scrapers or improvements. See [issues](https://github.com/slovensko-digital/ekosystem/issues) if you want to help, request a feature or file a bug. Join us on our [public slack channel](http://slack.slovensko.digital/) if you want to talk live. diff --git a/app.json b/app.json new file mode 100644 index 00000000..e57471a0 --- /dev/null +++ b/app.json @@ -0,0 +1,7 @@ +{ + "scripts": { + "dokku": { + "postdeploy": "bundle exec rake db:migrate" + } + } +} diff --git a/app/jobs/ra/fetch_changes_batch_job.rb b/app/jobs/ra/fetch_changes_batch_job.rb index 7a3b1a40..6193024c 100644 --- a/app/jobs/ra/fetch_changes_batch_job.rb +++ b/app/jobs/ra/fetch_changes_batch_job.rb @@ -1,5 +1,12 @@ +require 'harvester_utils/downloader' + module Ra class RecordBuilder + GENERIC_CHANGE_COLUMNS = [ + :changeId, :changedAt, :databaseOperation, :versionId, :createdReason, + :validFrom, :validTo, :effectiveDate + ] + def initialize @data = {} end @@ -17,7 +24,6 @@ def start_property_registration_change end def report_generic(tag) - # puts tag.inspect case tag.name when :propertyRegistrationNumberChange build_property_registration_number_change(tag) @@ -42,7 +48,7 @@ def report_generic(tag) def report_end if @data[:changes_id] - Ra::ChangesBatch.find_or_create_by!(id: @data[:changes_id], generated_at: Time.parse(@data[:generated_at])) + Ra::ChangesBatch.find_or_create_by!(id: @data[:changes_id], generated_at: Time.zone.parse(@data[:generated_at])) end end @@ -50,28 +56,16 @@ def report_end def build_property_registration_number_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = child.children.first.value.to_i - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:property_registration_number_id] = child.children.first.value.to_i - when :versionId - payload[:version_id] = child.children.first.value.to_i - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:property_registration_number_object] = Ra::PropertyRegistrationNumber.find_or_create_by!(id: id) when :PropertyRegistrationNumber - payload[:property_registration_number] = child.children.first.value.to_i + payload[:property_registration_number] = Integer(child.children.first.value) when :Building payload[:building_contains_flats] = child.attrs[:ContainsFlats] == 'true' child.children.each do |ch| @@ -80,20 +74,24 @@ def build_property_registration_number_change(tag) codelist = parse_codelist(ch.children.first) fail unless codelist[:code] == 'CL010142' if codelist[:item] - payload[:building_purpose] = Ra::BuildingPurpose.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + payload[:building_purpose_code] = Ra::BuildingPurposeCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) end when :BuildingTypeCode codelist = parse_codelist(ch.children.first) fail unless codelist[:code] == 'CL010143' - payload[:building_type] = Ra::BuildingType.find_or_create_by!(id: codelist[:item][:code], name: codelist[:item][:name]) + if codelist[:item] + payload[:building_type_code] = Ra::BuildingTypeCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + end else fail "Don't know how to handle #{ch.name}" end end when :municipalityIdentifier - payload[:municipality_id] = child.children.first.value.to_i + id = Integer(child.children.first.value) + payload[:municipality] = Ra::Municipality.find_or_create_by!(id: id) when :districtIdentifier - payload[:district_id] = child.children.first.value.to_i + id = Integer(child.children.first.value) + payload[:district] = Ra::District.find_or_create_by!(id: id) else fail "Don't know how to handle #{child.name}" end @@ -104,28 +102,16 @@ def build_property_registration_number_change(tag) def build_building_number_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:building_number_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:building_number_object] = Ra::BuildingNumber.find_or_create_by!(id: id) when :verifiedAt - payload[:verified_at] = Time.parse(child.children.first.value) + payload[:verified_at] = Time.zone.parse(child.children.first.value) when :BuildingNumber payload[:building_number] = child.children.first.value when :BuildingIndex @@ -135,9 +121,11 @@ def build_building_number_change(tag) when :AddressPoint payload[:address_point] = parse_address_point(child) when :propertyRegistrationNumberIdentifier - payload[:property_registration_number_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + payload[:property_registration_number] = Ra::PropertyRegistrationNumber.find_or_create_by!(id: id) when :streetNameIdentifier - payload[:street_name_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + payload[:street_name] = Ra::StreetName.find_or_create_by!(id: id) else fail "Don't know how to handle #{child.name}" end @@ -148,26 +136,14 @@ def build_building_number_change(tag) def build_building_unit_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:building_unit_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:building_unit_object] = Ra::BuildingUnit.find_or_create_by!(id: id) when :BuildingUnit payload[:building_unit_label] = child.children.first.value payload[:building_unit_floor] = child.attrs[:Floor] @@ -184,66 +160,51 @@ def build_building_unit_change(tag) def build_street_name_change(tag) payload = {} + municipalities_payload = [] + districts_payload = [] + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:street_name_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:street_name_object] = Ra::StreetName.find_or_create_by!(id: id) when :StreetName payload[:street_name] = child.children.first.value when :municipalityIdentifier - payload[:municipality_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + municipalities_payload << Ra::Municipality.find_or_create_by!(id: id) when :districtIdentifier - payload[:district_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + districts_payload << Ra::District.find_or_create_by!(id: id) else fail "Don't know how to handle #{child.name}" end end - Ra::StreetNameChange.find_or_create_by!(payload) + street_name_change = Ra::StreetNameChange.find_or_initialize_by(payload) + street_name_change.municipalities = municipalities_payload + street_name_change.districts = districts_payload + street_name_change.save! end def build_region_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:region_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:region_object] = Ra::Region.find_or_create_by!(id: id) when :Region codelist = parse_codelist(child.children.first) fail unless codelist[:code] == 'CL000023' - payload[:region_code] = Ra::RegionCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + if codelist[:item] + payload[:region_code] = Ra::RegionCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + end else fail "Don't know how to handle #{child.name}" end @@ -254,32 +215,23 @@ def build_region_change(tag) def build_county_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:county_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:county_object] = Ra::County.find_or_create_by!(id: id) when :regionIdentifier - payload[:region_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + payload[:region] = Ra::Region.find_or_create_by!(id: id) when :County codelist = parse_codelist(child.children.first) fail unless codelist[:code] == 'CL000024' - payload[:county_code] = Ra::CountyCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + if codelist[:item] + payload[:county_code] = Ra::CountyCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + end else fail "Don't know how to handle #{child.name}" end @@ -290,34 +242,28 @@ def build_county_change(tag) def build_municipality_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:municipality_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:municipality_object] = Ra::Municipality.find_or_create_by!(id: id) when :countyIdentifier - payload[:county_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + payload[:county] = Ra::County.find_or_create_by!(id: id) + when :cityIdentifier + id = Integer(child.children.first.value) + payload[:city_id] = id when :status payload[:municipality_status] = child.children.first.value when :Municipality codelist = parse_codelist(child.children.first) fail unless codelist[:code] == 'CL000025' - payload[:municipality_code] = Ra::MunicipalityCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + if codelist[:item] + payload[:municipality_code] = Ra::MunicipalityCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + end else fail "Don't know how to handle #{child.name}" end @@ -328,33 +274,24 @@ def build_municipality_change(tag) def build_district_change(tag) payload = {} + tag.children.each do |child| case child.name - when :changeId - payload[:id] = Integer(child.children.first.value) - when :changedAt - payload[:changed_at] = Time.parse(child.children.first.value) - when :databaseOperation - payload[:database_operation] = child.children.first.value + when *GENERIC_CHANGE_COLUMNS + payload.merge!(parse_generic_change_tag(child)) when :objectId - payload[:district_id] = Integer(child.children.first.value) - when :versionId - payload[:version_id] = Integer(child.children.first.value) - when :createdReason - payload[:created_reason] = child.children.first.value - when :validFrom - payload[:valid_from] = Time.parse(child.children.first.value) - when :validTo - payload[:valid_to] = Time.parse(child.children.first.value) - when :effectiveDate - payload[:effective_on] = Date.parse(child.children.first.value) + id = Integer(child.children.first.value) + payload[:district_object] = Ra::District.find_or_create_by!(id: id) when :municipalityIdentifier - payload[:municipality_id] = Integer(child.children.first.value) + id = Integer(child.children.first.value) + payload[:municipality] = Ra::Municipality.find_or_create_by!(id: id) when :District payload[:unique_numbering] = child.attrs[:UniqueNumbering] == 'true' codelist = parse_codelist(child.children.first) fail unless codelist[:code] == 'CL010141' - payload[:district_code] = Ra::DistrictCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + if codelist[:item] + payload[:district_code] = Ra::DistrictCode.find_or_create_by!(code: codelist[:item][:code], name: codelist[:item][:name]) + end else fail "Don't know how to handle #{child.name}" end @@ -363,6 +300,32 @@ def build_district_change(tag) Ra::DistrictChange.find_or_create_by!(payload) end + def parse_generic_change_tag(tag) + payload = {} + + case tag.name + when :changeId + id = Integer(tag.children.first.value) + payload[:change] = Ra::Change.find_or_create_by!(id: id) + when :changedAt + payload[:changed_at] = Time.zone.parse(tag.children.first.value) + when :databaseOperation + payload[:database_operation] = tag.children.first.value + when :versionId + payload[:version_id] = Integer(tag.children.first.value) + when :createdReason + payload[:created_reason] = tag.children.first.value + when :validFrom + payload[:valid_from] = Time.zone.parse(tag.children.first.value) + when :validTo + payload[:valid_to] = Time.zone.parse(tag.children.first.value) + when :effectiveDate + payload[:effective_on] = Date.parse(tag.children.first.value) + end + + payload + end + def parse_codelist(tag) codelist = {} tag.children.each do |child| @@ -531,8 +494,8 @@ def start_element(name, stack, listener) def end_element(name, stack, listener) case name when :'ns0:getChangesResponse' - when :'ns0:register' listener.report_end + when :'ns0:register' when :return when :corrId when :resultCode @@ -583,27 +546,22 @@ def initialize(listener) end def instruct(name) - # puts "instruct: #{name}" @stack.current.instruct(name, @stack, @listener) end def start_element(name) - # puts "start: #{name}" @stack.current.start_element(name, @stack, @listener) end def end_element(name) - # puts "end: #{name}" @stack.current.end_element(name, @stack, @listener) end def attr(name, value) - # puts " #{name} => #{value}" @stack.current.attr(name, value, @stack, @listener) end def text(value) - # puts "text #{value}" @stack.current.text(value, @stack, @listener) end end @@ -614,17 +572,25 @@ class Ra::FetchChangesBatchJob sidekiq_options queue: 'ra' - sidekiq_retry_in do |_count| - 1.day.to_i - end - - def perform(url, downloader: ::Harvester::Utils) + def perform(url, downloader: HarvesterUtils::Downloader) file = downloader.download_file(url) - perform_on_file(file) + perform_on_file(file.path) end def perform_on_file(path) handler = Sample.new(Ra::RecordBuilder.new) - Ox.sax_parse(handler, File.open(path, 'r')) + with_parsing_context do + Ox.sax_parse(handler, File.open(path, 'r')) + end + end + + private + + def with_parsing_context(&block) + Ra::ChangesBatch.transaction do + Time.use_zone("Europe/Bratislava") do + yield + end + end end end diff --git a/app/jobs/ra/fetch_initial_batch_job.rb b/app/jobs/ra/fetch_initial_batch_job.rb new file mode 100644 index 00000000..314c003c --- /dev/null +++ b/app/jobs/ra/fetch_initial_batch_job.rb @@ -0,0 +1,42 @@ +require 'zip' +require 'harvester_utils/downloader' + +module Ra + class FetchInitialBatchJob + include Sidekiq::Worker + include Sidekiq::Symbols + + sidekiq_options queue: 'ra' + + INITIAL_BATCH_URL = 'https://data.gov.sk/dataset/de3dd18f-9124-4acb-ae00-705555332256/resource/a904effb-7c5c-43ce-8c59-270932e6a19a/download/bbpradavkainit20151009prod.zip' + + def perform + initial_batch_zip = HarvesterUtils::Downloader.download_file(INITIAL_BATCH_URL) + + for_each_file_in_zip(initial_batch_zip.path) do |extracted_file| + Ra::FetchChangesBatchJob.new.perform_on_file(extracted_file) + end + end + + private + + def for_each_file_in_zip(zip_file, &block) + Zip::File.open(zip_file) do |zip_file| + zip_file.each do |entry| + logger.debug "Extracting #{entry.name}" + + extracted_file = Tempfile.new(entry.name) + begin + extracted_file.binmode + extracted_file.write(entry.get_input_stream.read) + + yield extracted_file + ensure + extracted_file.close + extracted_file.unlink + end + end + end + end + end +end diff --git a/app/jobs/ra/sync_changes_batches_job.rb b/app/jobs/ra/sync_changes_batches_job.rb index cd0a3b0b..c528c9a9 100644 --- a/app/jobs/ra/sync_changes_batches_job.rb +++ b/app/jobs/ra/sync_changes_batches_job.rb @@ -4,21 +4,27 @@ class Ra::SyncChangesBatchesJob sidekiq_options queue: 'ra' - sidekiq_retry_in do |_count| - 1.day.to_i - end - CHANGES_BATCHES_URL = 'https://data.gov.sk/dataset/register-adries-zmenove-davky' - def perform(downloader: Faraday, job: ::Ra::FetchChangesBatchJob) - html = downloader.get(CHANGES_BATCHES_URL).body - doc = Nokogiri::HTML(html) + @@downloader = Faraday.new(request: { timeout: 5.minutes.to_i }) + + def perform(downloader: @@downloader, job: ::Ra::FetchChangesBatchJob) + response = downloader.get(CHANGES_BATCHES_URL) + + response.status == 200 || fail("Received non-200 HTTP status: #{response.status}") + + doc = Nokogiri::HTML(response.body) doc.css('#dataset-resources .resource-item').each do |item| next unless item.at_css('a.heading')[:title].include?('Zmenová dávka č. ') download_link = item.at_css('a.resource-url-analytics') url = download_link[:href].gsub('https://MODMCASV222/', 'https://data.gov.sk/') - job.perform_async(url) + job.perform_async(url) if should_download_batch?(url) end end + + def should_download_batch?(batch_url) + batch_id = batch_url.match(/zmenovadavka(\d+)\.xml$/)[1] + !Ra::ChangesBatch.exists?(id: batch_id) + end end diff --git a/app/jobs/upvs/fetch_public_authority_edesks_list_job.rb b/app/jobs/upvs/fetch_public_authority_edesks_list_job.rb new file mode 100644 index 00000000..31946346 --- /dev/null +++ b/app/jobs/upvs/fetch_public_authority_edesks_list_job.rb @@ -0,0 +1,22 @@ +require 'csv' +require 'harvester_utils/downloader' + +class Upvs::FetchPublicAuthorityEdesksListJob < ApplicationJob + # TODO timeouts + queue_as :upvs + + def perform(downloader: HarvesterUtils::Downloader) + file = downloader.download_file('https://data.gov.sk/dataset/7d918796-3c9a-44a5-95a9-7e1dfca76407/resource/456a5bc6-32bd-4346-bcd1-12ee171ac010/download/zoznamintituciisozriadenoues.csv') + CSV.foreach(file, headers: true) do |row| + uri = row.fetch('URI') + edesk = Upvs::PublicAuthorityEdesk.find_or_initialize_by(uri: uri) + edesk.cin = row.fetch('IČO') + edesk.name = row.fetch('NAZOV INŠTITÚCIE') + edesk.street = row.fetch('ULICA') + edesk.street_number = row.fetch('ČÍSLO') + edesk.postal_code = row.fetch('PSČ') + edesk.city = row.fetch('MESTO') + edesk.save! + end + end +end diff --git a/app/lib/harvester/utils.rb b/app/lib/harvester/utils.rb deleted file mode 100644 index ef29bb16..00000000 --- a/app/lib/harvester/utils.rb +++ /dev/null @@ -1,16 +0,0 @@ -require 'typhoeus' -require 'tempfile' - -module Harvester - class Utils - def self.download_file(url) - response = Typhoeus.get(url) - fail unless response.success? - file = Tempfile.new(rand(1_000_000).to_s) - file.binmode - file.write(response.body) - file.close - file.path - end - end -end diff --git a/app/models/ra/building_number.rb b/app/models/ra/building_number.rb new file mode 100644 index 00000000..2a5e4e7b --- /dev/null +++ b/app/models/ra/building_number.rb @@ -0,0 +1,2 @@ +class Ra::BuildingNumber < ApplicationRecord +end diff --git a/app/models/ra/building_number_change.rb b/app/models/ra/building_number_change.rb index 8c8cc8d1..fbc3cebb 100644 --- a/app/models/ra/building_number_change.rb +++ b/app/models/ra/building_number_change.rb @@ -1,2 +1,10 @@ class Ra::BuildingNumberChange < ApplicationRecord + belongs_to :change, optional: true + belongs_to :building_number_object, + foreign_key: 'building_number_id', + class_name: 'Ra::BuildingNumber' + belongs_to :property_registration_number, + class_name: Ra::PropertyRegistrationNumber, + optional: true + belongs_to :street_name, class_name: Ra::StreetName, optional: true end diff --git a/app/models/ra/building_purpose.rb b/app/models/ra/building_purpose.rb deleted file mode 100644 index f246576b..00000000 --- a/app/models/ra/building_purpose.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Ra::BuildingPurpose < ApplicationRecord -end diff --git a/app/models/ra/building_purpose_code.rb b/app/models/ra/building_purpose_code.rb new file mode 100644 index 00000000..e38f06cf --- /dev/null +++ b/app/models/ra/building_purpose_code.rb @@ -0,0 +1,2 @@ +class Ra::BuildingPurposeCode < ApplicationRecord +end diff --git a/app/models/ra/building_type.rb b/app/models/ra/building_type.rb deleted file mode 100644 index 33977393..00000000 --- a/app/models/ra/building_type.rb +++ /dev/null @@ -1,2 +0,0 @@ -class Ra::BuildingType < ApplicationRecord -end diff --git a/app/models/ra/building_type_code.rb b/app/models/ra/building_type_code.rb new file mode 100644 index 00000000..ba7b8dfe --- /dev/null +++ b/app/models/ra/building_type_code.rb @@ -0,0 +1,2 @@ +class Ra::BuildingTypeCode < ApplicationRecord +end diff --git a/app/models/ra/building_unit.rb b/app/models/ra/building_unit.rb new file mode 100644 index 00000000..217540bf --- /dev/null +++ b/app/models/ra/building_unit.rb @@ -0,0 +1,2 @@ +class Ra::BuildingUnit < ApplicationRecord +end diff --git a/app/models/ra/building_unit_change.rb b/app/models/ra/building_unit_change.rb index b54d8c18..872265ac 100644 --- a/app/models/ra/building_unit_change.rb +++ b/app/models/ra/building_unit_change.rb @@ -1,2 +1,4 @@ class Ra::BuildingUnitChange < ApplicationRecord + belongs_to :change, optional: true + belongs_to :building_unit_object, foreign_key: 'building_unit_id', class_name: 'Ra::BuildingUnit' end diff --git a/app/models/ra/change.rb b/app/models/ra/change.rb new file mode 100644 index 00000000..bf47a1ff --- /dev/null +++ b/app/models/ra/change.rb @@ -0,0 +1,2 @@ +class Ra::Change < ApplicationRecord +end diff --git a/app/models/ra/county.rb b/app/models/ra/county.rb new file mode 100644 index 00000000..8a5015ae --- /dev/null +++ b/app/models/ra/county.rb @@ -0,0 +1,2 @@ +class Ra::County < ApplicationRecord +end diff --git a/app/models/ra/county_change.rb b/app/models/ra/county_change.rb index 473216ba..e62c0d04 100644 --- a/app/models/ra/county_change.rb +++ b/app/models/ra/county_change.rb @@ -1,3 +1,6 @@ class Ra::CountyChange < ApplicationRecord - belongs_to :county_code, :class_name => 'Ra::CountyCode' + belongs_to :change, optional: true + belongs_to :county_object, foreign_key: 'county_id', class_name: 'Ra::County' + belongs_to :region, class_name: 'Ra::Region', optional: true + belongs_to :county_code, class_name: 'Ra::CountyCode' end diff --git a/app/models/ra/district.rb b/app/models/ra/district.rb new file mode 100644 index 00000000..853da931 --- /dev/null +++ b/app/models/ra/district.rb @@ -0,0 +1,2 @@ +class Ra::District < ApplicationRecord +end diff --git a/app/models/ra/district_change.rb b/app/models/ra/district_change.rb index 49a91a8a..6b03e723 100644 --- a/app/models/ra/district_change.rb +++ b/app/models/ra/district_change.rb @@ -1,3 +1,6 @@ class Ra::DistrictChange < ApplicationRecord - belongs_to :district_code, :class_name => 'Ra::DistrictCode' + belongs_to :change, optional: true + belongs_to :district_object, foreign_key: 'district_id', class_name: 'Ra::District' + belongs_to :municipality, class_name: 'Ra::Municipality', optional: true + belongs_to :district_code, class_name: 'Ra::DistrictCode' end diff --git a/app/models/ra/municipality.rb b/app/models/ra/municipality.rb new file mode 100644 index 00000000..895370de --- /dev/null +++ b/app/models/ra/municipality.rb @@ -0,0 +1,2 @@ +class Ra::Municipality < ApplicationRecord +end diff --git a/app/models/ra/municipality_change.rb b/app/models/ra/municipality_change.rb index 4b544d91..848edc79 100644 --- a/app/models/ra/municipality_change.rb +++ b/app/models/ra/municipality_change.rb @@ -1,3 +1,6 @@ class Ra::MunicipalityChange < ApplicationRecord - belongs_to :municipality_code, :class_name => 'Ra::MunicipalityCode' + belongs_to :change, optional: true + belongs_to :municipality_object, foreign_key: 'municipality_id', class_name: 'Ra::Municipality' + belongs_to :county, class_name: 'Ra::County', optional: true + belongs_to :municipality_code, class_name: 'Ra::MunicipalityCode' end diff --git a/app/models/ra/property_registration_number.rb b/app/models/ra/property_registration_number.rb new file mode 100644 index 00000000..7cbc05e6 --- /dev/null +++ b/app/models/ra/property_registration_number.rb @@ -0,0 +1,2 @@ +class Ra::PropertyRegistrationNumber < ApplicationRecord +end diff --git a/app/models/ra/property_registration_number_change.rb b/app/models/ra/property_registration_number_change.rb index 263b5baa..30bb921c 100644 --- a/app/models/ra/property_registration_number_change.rb +++ b/app/models/ra/property_registration_number_change.rb @@ -1,4 +1,11 @@ class Ra::PropertyRegistrationNumberChange < ApplicationRecord - belongs_to :building_purpose, :class_name => 'Ra::BuildingPurpose', optional: true - belongs_to :building_type, :class_name => 'Ra::BuildingType', optional: true + belongs_to :change, optional: true + belongs_to :property_registration_number_object, + foreign_key: 'property_registration_number_id', + class_name: 'Ra::PropertyRegistrationNumber', + :required => true + belongs_to :municipality, class_name: 'Ra::Municipality', optional: true + belongs_to :district, class_name: 'Ra::District', optional: true + belongs_to :building_purpose_code, optional: true + belongs_to :building_type_code, optional: true end diff --git a/app/models/ra/region.rb b/app/models/ra/region.rb new file mode 100644 index 00000000..ed99835a --- /dev/null +++ b/app/models/ra/region.rb @@ -0,0 +1,2 @@ +class Ra::Region < ApplicationRecord +end diff --git a/app/models/ra/region_change.rb b/app/models/ra/region_change.rb index 5170185a..d2bda183 100644 --- a/app/models/ra/region_change.rb +++ b/app/models/ra/region_change.rb @@ -1,3 +1,5 @@ class Ra::RegionChange < ApplicationRecord - belongs_to :region_code, :class_name => 'Ra::RegionCode' + belongs_to :change, optional: true + belongs_to :region_object, foreign_key: 'region_id', class_name: 'Ra::Region' + belongs_to :region_code, class_name: 'Ra::RegionCode' end diff --git a/app/models/ra/street_name.rb b/app/models/ra/street_name.rb new file mode 100644 index 00000000..6fb6d6c6 --- /dev/null +++ b/app/models/ra/street_name.rb @@ -0,0 +1,2 @@ +class Ra::StreetName < ApplicationRecord +end diff --git a/app/models/ra/street_name_change.rb b/app/models/ra/street_name_change.rb index c953ce48..5d23d22f 100644 --- a/app/models/ra/street_name_change.rb +++ b/app/models/ra/street_name_change.rb @@ -1,2 +1,10 @@ class Ra::StreetNameChange < ApplicationRecord + belongs_to :change, optional: true + belongs_to :street_name_object, class_name: 'Ra::StreetName', foreign_key: 'street_name_id' + has_and_belongs_to_many :municipalities, + class_name: 'Ra::Municipality', + join_table: 'ra.street_name_changes_municipalities' + has_and_belongs_to_many :districts, + class_name: 'Ra::District', + join_table: 'ra.street_name_changes_districts' end diff --git a/app/models/upvs.rb b/app/models/upvs.rb new file mode 100644 index 00000000..e55d2dc8 --- /dev/null +++ b/app/models/upvs.rb @@ -0,0 +1,5 @@ +module Upvs + def self.table_name_prefix + 'upvs.' + end +end diff --git a/app/models/upvs/public_authority_edesk.rb b/app/models/upvs/public_authority_edesk.rb new file mode 100644 index 00000000..80dbfd34 --- /dev/null +++ b/app/models/upvs/public_authority_edesk.rb @@ -0,0 +1,2 @@ +class Upvs::PublicAuthorityEdesk < ApplicationRecord +end diff --git a/bin/rails b/bin/rails index 5badb2fd..07396602 100755 --- a/bin/rails +++ b/bin/rails @@ -1,9 +1,4 @@ #!/usr/bin/env ruby -begin - load File.expand_path('../spring', __FILE__) -rescue LoadError => e - raise unless e.message.include?('spring') -end APP_PATH = File.expand_path('../config/application', __dir__) require_relative '../config/boot' require 'rails/commands' diff --git a/bin/rake b/bin/rake index d87d5f57..17240489 100755 --- a/bin/rake +++ b/bin/rake @@ -1,9 +1,4 @@ #!/usr/bin/env ruby -begin - load File.expand_path('../spring', __FILE__) -rescue LoadError => e - raise unless e.message.include?('spring') -end require_relative '../config/boot' require 'rake' Rake.application.run diff --git a/circle.yml b/circle.yml new file mode 100644 index 00000000..c7e391be --- /dev/null +++ b/circle.yml @@ -0,0 +1,6 @@ +test: + override: + - RAILS_ENV=test bundle exec rspec -r rspec_junit_formatter --format progress --format RspecJunitFormatter -o $CIRCLE_TEST_REPORTS/rspec/junit.xml + post: + - bundle exec codeclimate-test-reporter + diff --git a/clock.rb b/clock.rb index 669d3d78..48c868eb 100644 --- a/clock.rb +++ b/clock.rb @@ -9,4 +9,5 @@ module Clockwork end every(1.day, 'ra:sync', at: '5:00') + every(1.week, 'upvs:public_authority_edesks:sync', at: 'Sunday 4:00') end diff --git a/config/application.rb b/config/application.rb index f4cb3f0e..cfddbcdd 100644 --- a/config/application.rb +++ b/config/application.rb @@ -5,11 +5,12 @@ require "active_model/railtie" require "active_job/railtie" require "active_record/railtie" -#require "action_controller/railtie" -# require "action_mailer/railtie" +require "action_controller/railtie" +require "action_mailer/railtie" require "action_view/railtie" require "action_cable/engine" -# require "sprockets/railtie" +require "action_cable/engine" +require "sprockets/railtie" # require "rails/test_unit/railtie" # Require the gems listed in Gemfile, including any gems diff --git a/config/database.yml b/config/database.yml index 7c7c0947..9a573501 100644 --- a/config/database.yml +++ b/config/database.yml @@ -1,6 +1,9 @@ default: &default adapter: postgis encoding: unicode + username: datahub + password: datahub + host: localhost # For details on connection pooling, see rails configuration guide # http://guides.rubyonrails.org/configuring.html#database-pooling pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 20 } %> @@ -8,22 +11,18 @@ default: &default development: <<: *default database: datahub_development - username: datahub - password: datahub # Warning: The database defined as "test" will be erased and # re-generated from your development database when you run "rake". # Do not set this db to the same as development or production. test: <<: *default - username: datahub - password: datahub database: datahub_test staging: <<: *default - password: <%= ENV['HARVESTER_DATABASE_URL'] %> + url: <%= ENV['DATABASE_URL'] %> production: <<: *default - password: <%= ENV['HARVESTER_DATABASE_URL'] %> + url: <%= ENV['DATABASE_URL'] %> diff --git a/config/environments/development.rb b/config/environments/development.rb index 2b9e7f1c..e64889cd 100644 --- a/config/environments/development.rb +++ b/config/environments/development.rb @@ -26,17 +26,29 @@ config.cache_store = :null_store end + # Don't care if the mailer can't send. + config.action_mailer.raise_delivery_errors = false + + config.action_mailer.perform_caching = false + # Print deprecation notices to the Rails logger. config.active_support.deprecation = :log # Raise an error on page load if there are pending migrations. config.active_record.migration_error = :page_load + # Debug mode disables concatenation and preprocessing of assets. + # This option may cause significant delays in view rendering with a large + # number of complex assets. + config.assets.debug = true + + # Suppress logger output for asset requests. + config.assets.quiet = true # Raises error for missing translations # config.action_view.raise_on_missing_translations = true # Use an evented file watcher to asynchronously detect changes in source code, # routes, locales, etc. This feature depends on the listen gem. - config.file_watcher = ActiveSupport::EventedFileUpdateChecker + # config.file_watcher = ActiveSupport::EventedFileUpdateChecker end diff --git a/config/environments/production.rb b/config/environments/production.rb index 67ed6726..35c2ed93 100644 --- a/config/environments/production.rb +++ b/config/environments/production.rb @@ -18,6 +18,14 @@ # Apache or NGINX already handles this. config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + # Compress JavaScripts and CSS. + config.assets.js_compressor = :uglifier + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = false + + # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb # Enable serving of images, stylesheets, and JavaScripts from an asset server. # config.action_controller.asset_host = 'http://assets.example.com' @@ -47,6 +55,11 @@ # Use a real queuing backend for Active Job (and separate queues per environment) # config.active_job.queue_adapter = :resque # config.active_job.queue_name_prefix = "harvester_#{Rails.env}" + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false # Enable locale fallbacks for I18n (makes lookups for any locale fall back to # the I18n.default_locale when a translation cannot be found). diff --git a/config/environments/staging.rb b/config/environments/staging.rb new file mode 100644 index 00000000..35c2ed93 --- /dev/null +++ b/config/environments/staging.rb @@ -0,0 +1,86 @@ +Rails.application.configure do + # Settings specified here will take precedence over those in config/application.rb. + + # Code is not reloaded between requests. + config.cache_classes = true + + # Eager load code on boot. This eager loads most of Rails and + # your application in memory, allowing both threaded web servers + # and those relying on copy on write to perform better. + # Rake tasks automatically ignore this option for performance. + config.eager_load = true + + # Full error reports are disabled and caching is turned on. + config.consider_all_requests_local = false + config.action_controller.perform_caching = true + + # Disable serving static files from the `/public` folder by default since + # Apache or NGINX already handles this. + config.public_file_server.enabled = ENV['RAILS_SERVE_STATIC_FILES'].present? + + # Compress JavaScripts and CSS. + config.assets.js_compressor = :uglifier + # config.assets.css_compressor = :sass + + # Do not fallback to assets pipeline if a precompiled asset is missed. + config.assets.compile = false + + # `config.assets.precompile` and `config.assets.version` have moved to config/initializers/assets.rb + + # Enable serving of images, stylesheets, and JavaScripts from an asset server. + # config.action_controller.asset_host = 'http://assets.example.com' + + # Specifies the header that your server uses for sending files. + # config.action_dispatch.x_sendfile_header = 'X-Sendfile' # for Apache + # config.action_dispatch.x_sendfile_header = 'X-Accel-Redirect' # for NGINX + + # Mount Action Cable outside main process or domain + # config.action_cable.mount_path = nil + # config.action_cable.url = 'wss://example.com/cable' + # config.action_cable.allowed_request_origins = [ 'http://example.com', /http:\/\/example.*/ ] + + # Force all access to the app over SSL, use Strict-Transport-Security, and use secure cookies. + # config.force_ssl = true + + # Use the lowest log level to ensure availability of diagnostic information + # when problems arise. + config.log_level = :debug + + # Prepend all log lines with the following tags. + config.log_tags = [ :request_id ] + + # Use a different cache store in production. + # config.cache_store = :mem_cache_store + + # Use a real queuing backend for Active Job (and separate queues per environment) + # config.active_job.queue_adapter = :resque + # config.active_job.queue_name_prefix = "harvester_#{Rails.env}" + config.action_mailer.perform_caching = false + + # Ignore bad email addresses and do not raise email delivery errors. + # Set this to true and configure the email server for immediate delivery to raise delivery errors. + # config.action_mailer.raise_delivery_errors = false + + # Enable locale fallbacks for I18n (makes lookups for any locale fall back to + # the I18n.default_locale when a translation cannot be found). + config.i18n.fallbacks = true + + # Send deprecation notices to registered listeners. + config.active_support.deprecation = :notify + + # Use default logging formatter so that PID and timestamp are not suppressed. + config.log_formatter = ::Logger::Formatter.new + + # Use a different logger for distributed setups. + # require 'syslog/logger' + # config.logger = ActiveSupport::TaggedLogging.new(Syslog::Logger.new 'app-name') + + if ENV["RAILS_LOG_TO_STDOUT"].present? + logger = ActiveSupport::Logger.new(STDOUT) + logger.formatter = config.log_formatter + config.logger = ActiveSupport::TaggedLogging.new(logger) + end + + # Do not dump schema after migrations. + config.active_record.dump_schema_after_migration = false +end diff --git a/config/environments/test.rb b/config/environments/test.rb index 60d9e0c2..30587ef6 100644 --- a/config/environments/test.rb +++ b/config/environments/test.rb @@ -27,6 +27,12 @@ # Disable request forgery protection in test environment. config.action_controller.allow_forgery_protection = false + config.action_mailer.perform_caching = false + + # Tell Action Mailer not to deliver emails to the real world. + # The :test delivery method accumulates sent emails in the + # ActionMailer::Base.deliveries array. + config.action_mailer.delivery_method = :test # Print deprecation notices to the stderr. config.active_support.deprecation = :stderr diff --git a/config/initializers/assets.rb b/config/initializers/assets.rb new file mode 100644 index 00000000..01ef3e66 --- /dev/null +++ b/config/initializers/assets.rb @@ -0,0 +1,11 @@ +# Be sure to restart your server when you modify this file. + +# Version of your assets, change this if you want to expire all your assets. +Rails.application.config.assets.version = '1.0' + +# Add additional assets to the asset load path +# Rails.application.config.assets.paths << Emoji.images_path + +# Precompile additional assets. +# application.js, application.css, and all non-JS/CSS in app/assets folder are already added. +# Rails.application.config.assets.precompile += %w( search.js ) diff --git a/config/initializers/new_framework_defaults.rb b/config/initializers/new_framework_defaults.rb index 0706cafd..4415dc3f 100644 --- a/config/initializers/new_framework_defaults.rb +++ b/config/initializers/new_framework_defaults.rb @@ -2,23 +2,22 @@ # # This file contains migration options to ease your Rails 5.0 upgrade. # -# Read the Rails 5.0 release notes for more info on each option. +# Once upgraded flip defaults one by one to migrate to the new default. +# +# Read the Guide for Upgrading Ruby on Rails for more info on each option. # Enable per-form CSRF tokens. Previous versions had false. -Rails.application.config.action_controller.per_form_csrf_tokens = true +Rails.application.config.action_controller.per_form_csrf_tokens = false # Enable origin-checking CSRF mitigation. Previous versions had false. -Rails.application.config.action_controller.forgery_protection_origin_check = true +Rails.application.config.action_controller.forgery_protection_origin_check = false # Make Ruby 2.4 preserve the timezone of the receiver when calling `to_time`. # Previous versions had false. -ActiveSupport.to_time_preserves_timezone = true +ActiveSupport.to_time_preserves_timezone = false # Require `belongs_to` associations by default. Previous versions had false. -Rails.application.config.active_record.belongs_to_required_by_default = true +Rails.application.config.active_record.belongs_to_required_by_default = false # Do not halt callback chains when a callback returns false. Previous versions had true. -ActiveSupport.halt_callback_chains_on_return_false = false - -# Configure SSL options to enable HSTS with subdomains. Previous versions had false. -Rails.application.config.ssl_options = { hsts: { subdomains: true } } +ActiveSupport.halt_callback_chains_on_return_false = true diff --git a/config/newrelic.yml b/config/newrelic.yml new file mode 100644 index 00000000..7bd895cc --- /dev/null +++ b/config/newrelic.yml @@ -0,0 +1,49 @@ +# +# This file configures the New Relic Agent. New Relic monitors Ruby, Java, +# .NET, PHP, Python and Node applications with deep visibility and low +# overhead. For more information, visit www.newrelic.com. +# +# Generated February 22, 2017 +# +# This configuration file is custom generated for Slovensko.Digital +# +# For full documentation of agent configuration options, please refer to +# https://docs.newrelic.com/docs/agents/ruby-agent/installation-configuration/ruby-agent-configuration + +common: &default_settings + # Required license key associated with your New Relic account. + license_key: <%= ENV['NEWRELIC_LICENSE_KEY'] %> + + # Your application name. Renaming here affects where data displays in New + # Relic. For more details, see https://docs.newrelic.com/docs/apm/new-relic-apm/maintenance/renaming-applications + app_name: Harvester + + # To disable the agent regardless of other settings, uncomment the following: + # agent_enabled: false + + # Logging level for log/newrelic_agent.log + log_level: info + + +# Environment-specific settings are in this section. +# RAILS_ENV or RACK_ENV (as appropriate) is used to determine the environment. +# If your application has other named environments, configure them here. +development: + <<: *default_settings + app_name: Harvester (Development) + + # NOTE: There is substantial overhead when running in developer mode. + # Do not use for production or load testing. + developer_mode: true + +test: + <<: *default_settings + # It doesn't make sense to report to New Relic from automated test runs. + monitor_mode: false + +staging: + <<: *default_settings + app_name: Harvester (Staging) + +production: + <<: *default_settings diff --git a/config/puma.rb b/config/puma.rb new file mode 100644 index 00000000..c7f311f8 --- /dev/null +++ b/config/puma.rb @@ -0,0 +1,47 @@ +# Puma can serve each request in a thread from an internal thread pool. +# The `threads` method setting takes two numbers a minimum and maximum. +# Any libraries that use thread pools should be configured to match +# the maximum value specified for Puma. Default is set to 5 threads for minimum +# and maximum, this matches the default thread size of Active Record. +# +threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }.to_i +threads threads_count, threads_count + +# Specifies the `port` that Puma will listen on to receive requests, default is 3000. +# +port ENV.fetch("PORT") { 3000 } + +# Specifies the `environment` that Puma will run in. +# +environment ENV.fetch("RAILS_ENV") { "development" } + +# Specifies the number of `workers` to boot in clustered mode. +# Workers are forked webserver processes. If using threads and workers together +# the concurrency of the application would be max `threads` * `workers`. +# Workers do not work on JRuby or Windows (both of which do not support +# processes). +# +# workers ENV.fetch("WEB_CONCURRENCY") { 2 } + +# Use the `preload_app!` method when specifying a `workers` number. +# This directive tells Puma to first boot the application and load code +# before forking the application. This takes advantage of Copy On Write +# process behavior so workers use less memory. If you use this option +# you need to make sure to reconnect any threads in the `on_worker_boot` +# block. +# +# preload_app! + +# The code in the `on_worker_boot` will be called if you are using +# clustered mode by specifying a number of `workers`. After each worker +# process is booted this block will be run, if you are using `preload_app!` +# option you will want to use this block to reconnect to any threads +# or connections that may have been created at application boot, Ruby +# cannot share connections between processes. +# +# on_worker_boot do +# ActiveRecord::Base.establish_connection if defined?(ActiveRecord) +# end + +# Allow puma to be restarted by `rails restart` command. +plugin :tmp_restart diff --git a/config/secrets.yml b/config/secrets.yml index eb5babbf..c11de0b4 100644 --- a/config/secrets.yml +++ b/config/secrets.yml @@ -11,10 +11,10 @@ # if you're sharing your code publicly. development: - secret_key_base: 0bcc66c8769f94fe0b9e26f12c6bce7e8e2f247d4ea4be5dab954626c59654d080cd342fdc03c5e04716233a6da8dcdc92493144347b7a8e72dc208b31c6cfb2 + secret_key_base: 8ac9998afcd81140a710d0771f447617c29b5706b8a486e3a738d62e71d97454d598af1bf9e7015ae6d5f8b0f5e82b1e61d5e5e7a7d0066df35dd1d1f11e31f0 test: - secret_key_base: 70345adc10de82c0181a068e57782e6611bb0565d381978d1611cdb641661b78efbbdfabc93f86676655abc8f328214ed72c15390b13f84a8c00c6b26a8c868e + secret_key_base: f38320776e7a833199315a349d66b41b3f167af88170753b5b82535f2a0d23ce9ea0455e4c9b562abe1b60756b1ffbbe0fe5c11854e18728daf9963b0fab4c97 # Do not keep production secrets in the repository, # instead read values from the environment. diff --git a/config/sidekiq.yml b/config/sidekiq.yml new file mode 100644 index 00000000..054e840e --- /dev/null +++ b/config/sidekiq.yml @@ -0,0 +1,7 @@ +--- +staging: + :concurrency: 8 +production: + :concurrency: 8 +:queues: + - [upvs, 1] \ No newline at end of file diff --git a/db/migrate/20170222131821_create_upvs_public_authority_edesks.rb b/db/migrate/20170222131821_create_upvs_public_authority_edesks.rb new file mode 100644 index 00000000..f3adbe1a --- /dev/null +++ b/db/migrate/20170222131821_create_upvs_public_authority_edesks.rb @@ -0,0 +1,24 @@ +class CreateUpvsPublicAuthorityEdesks < ActiveRecord::Migration[5.0] + def up + execute 'CREATE SCHEMA upvs' + + create_table 'upvs.public_authority_edesks' do |t| + t.integer :cin, null: false, limit: 8 + t.string :uri, null: false + t.string :name, null: false + t.string :street + t.string :street_number + t.string :postal_code + t.string :city + + t.timestamps + end + + add_index 'upvs.public_authority_edesks', :uri, unique: true + add_index 'upvs.public_authority_edesks', :cin + end + + def down + execute 'DROP SCHEMA upvs CASCADE' + end +end diff --git a/db/migrate/20170223074140_add_upvs_to_open_data_role.rb b/db/migrate/20170223074140_add_upvs_to_open_data_role.rb new file mode 100644 index 00000000..28f9df12 --- /dev/null +++ b/db/migrate/20170223074140_add_upvs_to_open_data_role.rb @@ -0,0 +1,6 @@ +class AddUpvsToOpenDataRole < ActiveRecord::Migration[5.0] + def up + execute 'GRANT USAGE ON SCHEMA upvs TO open_data' + execute 'GRANT SELECT ON ALL TABLES IN SCHEMA upvs TO open_data' + end +end diff --git a/db/migrate/20170430173505_create_ra_consolidated_models_stubs.rb b/db/migrate/20170430173505_create_ra_consolidated_models_stubs.rb new file mode 100644 index 00000000..1a4d8ca5 --- /dev/null +++ b/db/migrate/20170430173505_create_ra_consolidated_models_stubs.rb @@ -0,0 +1,35 @@ +class CreateRaConsolidatedModelsStubs < ActiveRecord::Migration[5.0] + def change + create_table 'ra.regions' do |t| + t.timestamps + end + + create_table 'ra.counties' do |t| + t.timestamps + end + + create_table 'ra.municipalities' do |t| + t.timestamps + end + + create_table 'ra.districts' do |t| + t.timestamps + end + + create_table 'ra.street_names' do |t| + t.timestamps + end + + create_table 'ra.property_registration_numbers' do |t| + t.timestamps + end + + create_table 'ra.building_numbers' do |t| + t.timestamps + end + + create_table 'ra.building_units' do |t| + t.timestamps + end + end +end diff --git a/db/migrate/20170430190843_create_ra_foreign_keys_with_consolidated_models_stubs.rb b/db/migrate/20170430190843_create_ra_foreign_keys_with_consolidated_models_stubs.rb new file mode 100644 index 00000000..5371be26 --- /dev/null +++ b/db/migrate/20170430190843_create_ra_foreign_keys_with_consolidated_models_stubs.rb @@ -0,0 +1,27 @@ +class CreateRaForeignKeysWithConsolidatedModelsStubs < ActiveRecord::Migration[5.0] + def change + add_foreign_key 'ra.region_changes', 'ra.regions', column: 'region_id' + + add_foreign_key 'ra.county_changes', 'ra.counties', column: 'county_id' + add_foreign_key 'ra.county_changes', 'ra.regions', column: 'region_id' + + add_foreign_key 'ra.municipality_changes', 'ra.municipalities', column: 'municipality_id' + add_foreign_key 'ra.municipality_changes', 'ra.counties', column: 'county_id' + add_reference 'ra.municipality_changes', :city, foreign_key: {to_table: 'ra.municipalities'} + + add_foreign_key 'ra.district_changes', 'ra.districts', column: 'district_id' + add_foreign_key 'ra.district_changes', 'ra.municipalities', column: 'municipality_id' + + add_foreign_key 'ra.street_name_changes', 'ra.street_names', column: 'street_name_id' + + add_foreign_key 'ra.property_registration_number_changes', 'ra.property_registration_numbers', column: 'property_registration_number_id' + add_foreign_key 'ra.property_registration_number_changes', 'ra.municipalities', column: 'municipality_id' + add_foreign_key 'ra.property_registration_number_changes', 'ra.districts', column: 'district_id' + + add_foreign_key 'ra.building_number_changes', 'ra.building_numbers', column: 'building_number_id' + add_foreign_key 'ra.building_number_changes', 'ra.property_registration_numbers', column: 'property_registration_number_id' + add_foreign_key 'ra.building_number_changes', 'ra.street_names', column: 'street_name_id' + + add_foreign_key 'ra.building_unit_changes', 'ra.building_units', column: 'building_unit_id' + end +end diff --git a/db/migrate/20170502120756_fix_ra_street_name_change_association_to_municipality_and_district.rb b/db/migrate/20170502120756_fix_ra_street_name_change_association_to_municipality_and_district.rb new file mode 100644 index 00000000..e968070a --- /dev/null +++ b/db/migrate/20170502120756_fix_ra_street_name_change_association_to_municipality_and_district.rb @@ -0,0 +1,20 @@ +class FixRaStreetNameChangeAssociationToMunicipalityAndDistrict < ActiveRecord::Migration[5.0] + def change + create_table 'ra.street_name_changes_municipalities', id: false do |t| + t.integer 'street_name_change_id', null: false + t.integer 'municipality_id', null: false + end + create_table 'ra.street_name_changes_districts', id: false do |t| + t.integer 'street_name_change_id', null: false + t.integer 'district_id', null: false + end + + remove_column 'ra.street_name_changes', :municipality_id + remove_column 'ra.street_name_changes', :district_id + + add_foreign_key 'ra.street_name_changes_municipalities', 'ra.street_name_changes', column: 'street_name_change_id' + add_foreign_key 'ra.street_name_changes_municipalities', 'ra.municipalities', column: 'municipality_id' + add_foreign_key 'ra.street_name_changes_districts', 'ra.street_name_changes', column: 'street_name_change_id' + add_foreign_key 'ra.street_name_changes_districts', 'ra.districts', column: 'district_id' + end +end diff --git a/db/migrate/20170503112646_create_ra_changes_and_change_id_columns.rb b/db/migrate/20170503112646_create_ra_changes_and_change_id_columns.rb new file mode 100644 index 00000000..ded5eaa0 --- /dev/null +++ b/db/migrate/20170503112646_create_ra_changes_and_change_id_columns.rb @@ -0,0 +1,14 @@ +class CreateRaChangesAndChangeIdColumns < ActiveRecord::Migration[5.0] + def change + create_table 'ra.changes' + + add_reference 'ra.region_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.county_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.municipality_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.district_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.street_name_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.property_registration_number_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.building_number_changes', :change, foreign_key: {to_table: 'ra.changes'} + add_reference 'ra.building_unit_changes', :change, foreign_key: {to_table: 'ra.changes'} + end +end diff --git a/db/migrate/20170503203539_unify_ra_codelist_tables.rb b/db/migrate/20170503203539_unify_ra_codelist_tables.rb new file mode 100644 index 00000000..a3df93f2 --- /dev/null +++ b/db/migrate/20170503203539_unify_ra_codelist_tables.rb @@ -0,0 +1,31 @@ +class UnifyRaCodelistTables < ActiveRecord::Migration[5.0] + def change + drop_table 'ra.building_types' + create_table 'ra.building_type_codes' do |t| + t.string :code + t.string :name, null: false + t.timestamps + end + add_index 'ra.building_type_codes', :code, unique: true + rename_column 'ra.property_registration_number_changes', 'building_type_id', 'building_type_code_id' + + drop_table 'ra.building_purposes' + create_table 'ra.building_purpose_codes' do |t| + t.string :code + t.string :name, null: false + t.timestamps + end + add_index 'ra.building_purpose_codes', :code, unique: true + rename_column 'ra.property_registration_number_changes', 'building_purpose_id', 'building_purpose_code_id' + + add_index 'ra.municipality_codes', :code, unique: true + add_index 'ra.district_codes', :code, unique: true + + add_foreign_key 'ra.region_changes', 'ra.region_codes', column: 'region_code_id' + add_foreign_key 'ra.county_changes', 'ra.county_codes', column: 'county_code_id' + add_foreign_key 'ra.municipality_changes', 'ra.municipality_codes', column: 'municipality_code_id' + add_foreign_key 'ra.district_changes', 'ra.district_codes', column: 'district_code_id' + add_foreign_key 'ra.property_registration_number_changes', 'ra.building_type_codes', column: 'building_type_code_id' + add_foreign_key 'ra.property_registration_number_changes', 'ra.building_purpose_codes', column: 'building_purpose_code_id' + end +end diff --git a/db/migrate/20170506104022_ra_drop_unique_indices_for_codes_tables.rb b/db/migrate/20170506104022_ra_drop_unique_indices_for_codes_tables.rb new file mode 100644 index 00000000..ec9a1ada --- /dev/null +++ b/db/migrate/20170506104022_ra_drop_unique_indices_for_codes_tables.rb @@ -0,0 +1,24 @@ +class RaDropUniqueIndicesForCodesTables < ActiveRecord::Migration[5.0] + def up + execute <<-SQL + SET search_path = ra; + + DROP INDEX "index_ra.region_codes_on_code"; + DROP INDEX "index_ra.county_codes_on_code"; + DROP INDEX "index_ra.district_codes_on_code"; + DROP INDEX "index_ra.municipality_codes_on_code"; + DROP INDEX "index_ra.building_purpose_codes_on_code"; + DROP INDEX "index_ra.building_type_codes_on_code"; + + SET search_path TO "$user", public; + SQL + end + def down + add_index 'ra.region_codes', :code, unique: true + add_index 'ra.county_codes', :code, unique: true + add_index 'ra.district_codes', :code, unique: true + add_index 'ra.municipality_codes', :code, unique: true + add_index 'ra.building_purpose_codes', :code, unique: true + add_index 'ra.building_type_codes', :code, unique: true + end +end diff --git a/db/migrate/20170506190014_ra_add_indices_to_street_name_changes_join_tables.rb b/db/migrate/20170506190014_ra_add_indices_to_street_name_changes_join_tables.rb new file mode 100644 index 00000000..977dd181 --- /dev/null +++ b/db/migrate/20170506190014_ra_add_indices_to_street_name_changes_join_tables.rb @@ -0,0 +1,8 @@ +class RaAddIndicesToStreetNameChangesJoinTables < ActiveRecord::Migration[5.0] + def change + add_index 'ra.street_name_changes_municipalities', 'street_name_change_id', name: 'street_name_changes_municipalities_change' + add_index 'ra.street_name_changes_municipalities', 'municipality_id', name: 'street_name_changes_municipalities_municipality' + add_index 'ra.street_name_changes_districts', 'street_name_change_id', name: 'street_name_changes_districts_change' + add_index 'ra.street_name_changes_districts', 'district_id', name: 'street_name_changes_districts_district' + end +end diff --git a/db/migrate/20170513200748_ra_add_effective_on_to_municipality_changes.rb b/db/migrate/20170513200748_ra_add_effective_on_to_municipality_changes.rb new file mode 100644 index 00000000..7f8e8184 --- /dev/null +++ b/db/migrate/20170513200748_ra_add_effective_on_to_municipality_changes.rb @@ -0,0 +1,5 @@ +class RaAddEffectiveOnToMunicipalityChanges < ActiveRecord::Migration[5.0] + def change + add_column 'ra.municipality_changes', 'effective_on', :date + end +end diff --git a/db/structure.sql b/db/structure.sql index 7dfd32ba..a0e7bb52 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2,8 +2,8 @@ -- PostgreSQL database dump -- --- Dumped from database version 9.5.4 --- Dumped by pg_dump version 9.5.4 +-- Dumped from database version 9.5.6 +-- Dumped by pg_dump version 9.5.6 SET statement_timeout = 0; SET lock_timeout = 0; @@ -97,6 +97,13 @@ CREATE SCHEMA ruz; CREATE SCHEMA socpoist; +-- +-- Name: upvs; Type: SCHEMA; Schema: -; Owner: - +-- + +CREATE SCHEMA upvs; + + -- -- Name: vvo; Type: SCHEMA; Schema: -; Owner: - -- @@ -122,7 +129,7 @@ COMMENT ON EXTENSION plpgsql IS 'PL/pgSQL procedural language'; -- Name: postgis; Type: EXTENSION; Schema: -; Owner: - -- -CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA datahub; +CREATE EXTENSION IF NOT EXISTS postgis WITH SCHEMA public; -- @@ -1126,9 +1133,10 @@ CREATE TABLE building_number_changes ( building_number character varying, building_index character varying NOT NULL, postal_code integer, - address_point datahub.geography(Point,4326), + address_point public.geography(Point,4326), created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1152,12 +1160,42 @@ ALTER SEQUENCE building_number_changes_id_seq OWNED BY building_number_changes.i -- --- Name: building_purposes; Type: TABLE; Schema: ra; Owner: - +-- Name: building_numbers; Type: TABLE; Schema: ra; Owner: - -- -CREATE TABLE building_purposes ( +CREATE TABLE building_numbers ( id integer NOT NULL, - code character varying NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: building_numbers_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE building_numbers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: building_numbers_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE building_numbers_id_seq OWNED BY building_numbers.id; + + +-- +-- Name: building_purpose_codes; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE building_purpose_codes ( + id integer NOT NULL, + code character varying, name character varying NOT NULL, created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL @@ -1165,10 +1203,10 @@ CREATE TABLE building_purposes ( -- --- Name: building_purposes_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- Name: building_purpose_codes_id_seq; Type: SEQUENCE; Schema: ra; Owner: - -- -CREATE SEQUENCE building_purposes_id_seq +CREATE SEQUENCE building_purpose_codes_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -1177,18 +1215,19 @@ CREATE SEQUENCE building_purposes_id_seq -- --- Name: building_purposes_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- Name: building_purpose_codes_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - -- -ALTER SEQUENCE building_purposes_id_seq OWNED BY building_purposes.id; +ALTER SEQUENCE building_purpose_codes_id_seq OWNED BY building_purpose_codes.id; -- --- Name: building_types; Type: TABLE; Schema: ra; Owner: - +-- Name: building_type_codes; Type: TABLE; Schema: ra; Owner: - -- -CREATE TABLE building_types ( +CREATE TABLE building_type_codes ( id integer NOT NULL, + code character varying, name character varying NOT NULL, created_at timestamp without time zone NOT NULL, updated_at timestamp without time zone NOT NULL @@ -1196,10 +1235,10 @@ CREATE TABLE building_types ( -- --- Name: building_types_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- Name: building_type_codes_id_seq; Type: SEQUENCE; Schema: ra; Owner: - -- -CREATE SEQUENCE building_types_id_seq +CREATE SEQUENCE building_type_codes_id_seq START WITH 1 INCREMENT BY 1 NO MINVALUE @@ -1208,10 +1247,10 @@ CREATE SEQUENCE building_types_id_seq -- --- Name: building_types_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- Name: building_type_codes_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - -- -ALTER SEQUENCE building_types_id_seq OWNED BY building_types.id; +ALTER SEQUENCE building_type_codes_id_seq OWNED BY building_type_codes.id; -- @@ -1233,7 +1272,8 @@ CREATE TABLE building_unit_changes ( building_unit_number character varying NOT NULL, building_unit_label character varying NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1256,6 +1296,45 @@ CREATE SEQUENCE building_unit_changes_id_seq ALTER SEQUENCE building_unit_changes_id_seq OWNED BY building_unit_changes.id; +-- +-- Name: building_units; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE building_units ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: building_units_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE building_units_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: building_units_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE building_units_id_seq OWNED BY building_units.id; + + +-- +-- Name: changes; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE changes ( + id integer NOT NULL +); + + -- -- Name: changes_batches; Type: TABLE; Schema: ra; Owner: - -- @@ -1287,6 +1366,55 @@ CREATE SEQUENCE changes_batches_id_seq ALTER SEQUENCE changes_batches_id_seq OWNED BY changes_batches.id; +-- +-- Name: changes_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE changes_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: changes_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE changes_id_seq OWNED BY changes.id; + + +-- +-- Name: counties; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE counties ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: counties_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE counties_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: counties_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE counties_id_seq OWNED BY counties.id; + + -- -- Name: county_changes; Type: TABLE; Schema: ra; Owner: - -- @@ -1304,7 +1432,8 @@ CREATE TABLE county_changes ( effective_on date NOT NULL, county_code_id integer NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1377,7 +1506,8 @@ CREATE TABLE district_changes ( district_code_id integer NOT NULL, unique_numbering boolean NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1432,6 +1562,66 @@ CREATE SEQUENCE district_codes_id_seq ALTER SEQUENCE district_codes_id_seq OWNED BY district_codes.id; +-- +-- Name: districts; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE districts ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: districts_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE districts_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: districts_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE districts_id_seq OWNED BY districts.id; + + +-- +-- Name: municipalities; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE municipalities ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: municipalities_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE municipalities_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: municipalities_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE municipalities_id_seq OWNED BY municipalities.id; + + -- -- Name: municipality_changes; Type: TABLE; Schema: ra; Owner: - -- @@ -1449,7 +1639,10 @@ CREATE TABLE municipality_changes ( municipality_code_id integer NOT NULL, municipality_status municipality_status, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + city_id integer, + change_id integer, + effective_on date ); @@ -1520,12 +1713,13 @@ CREATE TABLE property_registration_number_changes ( effective_on date, property_registration_number integer, building_contains_flats boolean NOT NULL, - building_purpose_id integer, - building_type_id integer, + building_purpose_code_id integer, + building_type_code_id integer, municipality_id integer NOT NULL, district_id integer, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1548,6 +1742,36 @@ CREATE SEQUENCE property_registration_number_changes_id_seq ALTER SEQUENCE property_registration_number_changes_id_seq OWNED BY property_registration_number_changes.id; +-- +-- Name: property_registration_numbers; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE property_registration_numbers ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: property_registration_numbers_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE property_registration_numbers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: property_registration_numbers_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE property_registration_numbers_id_seq OWNED BY property_registration_numbers.id; + + -- -- Name: region_changes; Type: TABLE; Schema: ra; Owner: - -- @@ -1564,7 +1788,8 @@ CREATE TABLE region_changes ( effective_on date NOT NULL, region_code_id integer NOT NULL, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer ); @@ -1619,6 +1844,36 @@ CREATE SEQUENCE region_codes_id_seq ALTER SEQUENCE region_codes_id_seq OWNED BY region_codes.id; +-- +-- Name: regions; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE regions ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: regions_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE regions_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: regions_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE regions_id_seq OWNED BY regions.id; + + -- -- Name: street_name_changes; Type: TABLE; Schema: ra; Owner: - -- @@ -1626,8 +1881,6 @@ ALTER SEQUENCE region_codes_id_seq OWNED BY region_codes.id; CREATE TABLE street_name_changes ( id integer NOT NULL, street_name_id integer NOT NULL, - municipality_id integer NOT NULL, - district_id integer, changed_at timestamp without time zone, database_operation change_type, version_id integer NOT NULL, @@ -1637,7 +1890,18 @@ CREATE TABLE street_name_changes ( effective_on date, street_name character varying, created_at timestamp without time zone NOT NULL, - updated_at timestamp without time zone NOT NULL + updated_at timestamp without time zone NOT NULL, + change_id integer +); + + +-- +-- Name: street_name_changes_districts; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE street_name_changes_districts ( + street_name_change_id integer NOT NULL, + district_id integer NOT NULL ); @@ -1660,6 +1924,46 @@ CREATE SEQUENCE street_name_changes_id_seq ALTER SEQUENCE street_name_changes_id_seq OWNED BY street_name_changes.id; +-- +-- Name: street_name_changes_municipalities; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE street_name_changes_municipalities ( + street_name_change_id integer NOT NULL, + municipality_id integer NOT NULL +); + + +-- +-- Name: street_names; Type: TABLE; Schema: ra; Owner: - +-- + +CREATE TABLE street_names ( + id integer NOT NULL, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: street_names_id_seq; Type: SEQUENCE; Schema: ra; Owner: - +-- + +CREATE SEQUENCE street_names_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: street_names_id_seq; Type: SEQUENCE OWNED BY; Schema: ra; Owner: - +-- + +ALTER SEQUENCE street_names_id_seq OWNED BY street_names.id; + + SET search_path = rpo, pg_catalog; -- @@ -2918,10 +3222,49 @@ CREATE SEQUENCE debts_lists_id_seq ALTER SEQUENCE debts_lists_id_seq OWNED BY debts_lists.id; -SET search_path = vvo, pg_catalog; +SET search_path = upvs, pg_catalog; -- --- Name: notice_types; Type: TABLE; Schema: vvo; Owner: - +-- Name: public_authority_edesks; Type: TABLE; Schema: upvs; Owner: - +-- + +CREATE TABLE public_authority_edesks ( + id integer NOT NULL, + cin bigint NOT NULL, + uri character varying NOT NULL, + name character varying NOT NULL, + street character varying, + street_number character varying, + postal_code character varying, + city character varying, + created_at timestamp without time zone NOT NULL, + updated_at timestamp without time zone NOT NULL +); + + +-- +-- Name: public_authority_edesks_id_seq; Type: SEQUENCE; Schema: upvs; Owner: - +-- + +CREATE SEQUENCE public_authority_edesks_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: public_authority_edesks_id_seq; Type: SEQUENCE OWNED BY; Schema: upvs; Owner: - +-- + +ALTER SEQUENCE public_authority_edesks_id_seq OWNED BY public_authority_edesks.id; + + +SET search_path = vvo, pg_catalog; + +-- +-- Name: notice_types; Type: TABLE; Schema: vvo; Owner: - -- CREATE TABLE notice_types ( @@ -3379,14 +3722,21 @@ ALTER TABLE ONLY building_number_changes ALTER COLUMN id SET DEFAULT nextval('bu -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- -ALTER TABLE ONLY building_purposes ALTER COLUMN id SET DEFAULT nextval('building_purposes_id_seq'::regclass); +ALTER TABLE ONLY building_numbers ALTER COLUMN id SET DEFAULT nextval('building_numbers_id_seq'::regclass); -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- -ALTER TABLE ONLY building_types ALTER COLUMN id SET DEFAULT nextval('building_types_id_seq'::regclass); +ALTER TABLE ONLY building_purpose_codes ALTER COLUMN id SET DEFAULT nextval('building_purpose_codes_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_type_codes ALTER COLUMN id SET DEFAULT nextval('building_type_codes_id_seq'::regclass); -- @@ -3396,6 +3746,20 @@ ALTER TABLE ONLY building_types ALTER COLUMN id SET DEFAULT nextval('building_ty ALTER TABLE ONLY building_unit_changes ALTER COLUMN id SET DEFAULT nextval('building_unit_changes_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_units ALTER COLUMN id SET DEFAULT nextval('building_units_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY changes ALTER COLUMN id SET DEFAULT nextval('changes_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- @@ -3403,6 +3767,13 @@ ALTER TABLE ONLY building_unit_changes ALTER COLUMN id SET DEFAULT nextval('buil ALTER TABLE ONLY changes_batches ALTER COLUMN id SET DEFAULT nextval('changes_batches_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY counties ALTER COLUMN id SET DEFAULT nextval('counties_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- @@ -3431,6 +3802,20 @@ ALTER TABLE ONLY district_changes ALTER COLUMN id SET DEFAULT nextval('district_ ALTER TABLE ONLY district_codes ALTER COLUMN id SET DEFAULT nextval('district_codes_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY districts ALTER COLUMN id SET DEFAULT nextval('districts_id_seq'::regclass); + + +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipalities ALTER COLUMN id SET DEFAULT nextval('municipalities_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- @@ -3452,6 +3837,13 @@ ALTER TABLE ONLY municipality_codes ALTER COLUMN id SET DEFAULT nextval('municip ALTER TABLE ONLY property_registration_number_changes ALTER COLUMN id SET DEFAULT nextval('property_registration_number_changes_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_numbers ALTER COLUMN id SET DEFAULT nextval('property_registration_numbers_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- @@ -3466,6 +3858,13 @@ ALTER TABLE ONLY region_changes ALTER COLUMN id SET DEFAULT nextval('region_chan ALTER TABLE ONLY region_codes ALTER COLUMN id SET DEFAULT nextval('region_codes_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY regions ALTER COLUMN id SET DEFAULT nextval('regions_id_seq'::regclass); + + -- -- Name: id; Type: DEFAULT; Schema: ra; Owner: - -- @@ -3473,6 +3872,13 @@ ALTER TABLE ONLY region_codes ALTER COLUMN id SET DEFAULT nextval('region_codes_ ALTER TABLE ONLY street_name_changes ALTER COLUMN id SET DEFAULT nextval('street_name_changes_id_seq'::regclass); +-- +-- Name: id; Type: DEFAULT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_names ALTER COLUMN id SET DEFAULT nextval('street_names_id_seq'::regclass); + + SET search_path = rpo, pg_catalog; -- @@ -3724,6 +4130,15 @@ ALTER TABLE ONLY debtors ALTER COLUMN id SET DEFAULT nextval('debtors_id_seq'::r ALTER TABLE ONLY debts_lists ALTER COLUMN id SET DEFAULT nextval('debts_lists_id_seq'::regclass); +SET search_path = upvs, pg_catalog; + +-- +-- Name: id; Type: DEFAULT; Schema: upvs; Owner: - +-- + +ALTER TABLE ONLY public_authority_edesks ALTER COLUMN id SET DEFAULT nextval('public_authority_edesks_id_seq'::regclass); + + SET search_path = vvo, pg_catalog; -- @@ -4009,19 +4424,27 @@ ALTER TABLE ONLY building_number_changes -- --- Name: building_purposes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- Name: building_numbers_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_numbers + ADD CONSTRAINT building_numbers_pkey PRIMARY KEY (id); + + +-- +-- Name: building_purpose_codes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- -ALTER TABLE ONLY building_purposes - ADD CONSTRAINT building_purposes_pkey PRIMARY KEY (id); +ALTER TABLE ONLY building_purpose_codes + ADD CONSTRAINT building_purpose_codes_pkey PRIMARY KEY (id); -- --- Name: building_types_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- Name: building_type_codes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- -ALTER TABLE ONLY building_types - ADD CONSTRAINT building_types_pkey PRIMARY KEY (id); +ALTER TABLE ONLY building_type_codes + ADD CONSTRAINT building_type_codes_pkey PRIMARY KEY (id); -- @@ -4032,6 +4455,14 @@ ALTER TABLE ONLY building_unit_changes ADD CONSTRAINT building_unit_changes_pkey PRIMARY KEY (id); +-- +-- Name: building_units_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_units + ADD CONSTRAINT building_units_pkey PRIMARY KEY (id); + + -- -- Name: changes_batches_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- @@ -4040,6 +4471,22 @@ ALTER TABLE ONLY changes_batches ADD CONSTRAINT changes_batches_pkey PRIMARY KEY (id); +-- +-- Name: changes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY changes + ADD CONSTRAINT changes_pkey PRIMARY KEY (id); + + +-- +-- Name: counties_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY counties + ADD CONSTRAINT counties_pkey PRIMARY KEY (id); + + -- -- Name: county_changes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- @@ -4072,6 +4519,22 @@ ALTER TABLE ONLY district_codes ADD CONSTRAINT district_codes_pkey PRIMARY KEY (id); +-- +-- Name: districts_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY districts + ADD CONSTRAINT districts_pkey PRIMARY KEY (id); + + +-- +-- Name: municipalities_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipalities + ADD CONSTRAINT municipalities_pkey PRIMARY KEY (id); + + -- -- Name: municipality_changes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- @@ -4096,6 +4559,14 @@ ALTER TABLE ONLY property_registration_number_changes ADD CONSTRAINT property_registration_number_changes_pkey PRIMARY KEY (id); +-- +-- Name: property_registration_numbers_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_numbers + ADD CONSTRAINT property_registration_numbers_pkey PRIMARY KEY (id); + + -- -- Name: region_changes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- @@ -4112,6 +4583,14 @@ ALTER TABLE ONLY region_codes ADD CONSTRAINT region_codes_pkey PRIMARY KEY (id); +-- +-- Name: regions_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY regions + ADD CONSTRAINT regions_pkey PRIMARY KEY (id); + + -- -- Name: street_name_changes_pkey; Type: CONSTRAINT; Schema: ra; Owner: - -- @@ -4120,6 +4599,14 @@ ALTER TABLE ONLY street_name_changes ADD CONSTRAINT street_name_changes_pkey PRIMARY KEY (id); +-- +-- Name: street_names_pkey; Type: CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_names + ADD CONSTRAINT street_names_pkey PRIMARY KEY (id); + + SET search_path = rpo, pg_catalog; -- @@ -4414,6 +4901,16 @@ ALTER TABLE ONLY debtors ADD CONSTRAINT socpoist_debtor_name UNIQUE (name, address, city); +SET search_path = upvs, pg_catalog; + +-- +-- Name: public_authority_edesks_pkey; Type: CONSTRAINT; Schema: upvs; Owner: - +-- + +ALTER TABLE ONLY public_authority_edesks + ADD CONSTRAINT public_authority_edesks_pkey PRIMARY KEY (id); + + SET search_path = vvo, pg_catalog; -- @@ -4781,6 +5278,13 @@ CREATE UNIQUE INDEX unique_schema_migrations ON schema_migrations USING btree (v SET search_path = ra, pg_catalog; +-- +-- Name: index_ra.building_number_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.building_number_changes_on_change_id" ON building_number_changes USING btree (change_id); + + -- -- Name: index_ra.building_number_changes_on_version_id; Type: INDEX; Schema: ra; Owner: - -- @@ -4789,17 +5293,24 @@ CREATE INDEX "index_ra.building_number_changes_on_version_id" ON building_number -- --- Name: index_ra.building_purposes_on_code; Type: INDEX; Schema: ra; Owner: - +-- Name: index_ra.building_unit_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - -- -CREATE UNIQUE INDEX "index_ra.building_purposes_on_code" ON building_purposes USING btree (code); +CREATE INDEX "index_ra.building_unit_changes_on_change_id" ON building_unit_changes USING btree (change_id); -- --- Name: index_ra.county_codes_on_code; Type: INDEX; Schema: ra; Owner: - +-- Name: index_ra.county_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - -- -CREATE UNIQUE INDEX "index_ra.county_codes_on_code" ON county_codes USING btree (code); +CREATE INDEX "index_ra.county_changes_on_change_id" ON county_changes USING btree (change_id); + + +-- +-- Name: index_ra.district_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.district_changes_on_change_id" ON district_changes USING btree (change_id); -- @@ -4809,6 +5320,20 @@ CREATE UNIQUE INDEX "index_ra.county_codes_on_code" ON county_codes USING btree CREATE UNIQUE INDEX "index_ra.district_codes_on_code_and_name" ON district_codes USING btree (code, name); +-- +-- Name: index_ra.municipality_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.municipality_changes_on_change_id" ON municipality_changes USING btree (change_id); + + +-- +-- Name: index_ra.municipality_changes_on_city_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.municipality_changes_on_city_id" ON municipality_changes USING btree (city_id); + + -- -- Name: index_ra.municipality_codes_on_code_and_name; Type: INDEX; Schema: ra; Owner: - -- @@ -4816,6 +5341,13 @@ CREATE UNIQUE INDEX "index_ra.district_codes_on_code_and_name" ON district_codes CREATE UNIQUE INDEX "index_ra.municipality_codes_on_code_and_name" ON municipality_codes USING btree (code, name); +-- +-- Name: index_ra.property_registration_number_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.property_registration_number_changes_on_change_id" ON property_registration_number_changes USING btree (change_id); + + -- -- Name: index_ra.property_registration_number_changes_on_version_id; Type: INDEX; Schema: ra; Owner: - -- @@ -4824,10 +5356,45 @@ CREATE INDEX "index_ra.property_registration_number_changes_on_version_id" ON pr -- --- Name: index_ra.region_codes_on_code; Type: INDEX; Schema: ra; Owner: - +-- Name: index_ra.region_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.region_changes_on_change_id" ON region_changes USING btree (change_id); + + +-- +-- Name: index_ra.street_name_changes_on_change_id; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX "index_ra.street_name_changes_on_change_id" ON street_name_changes USING btree (change_id); + + +-- +-- Name: street_name_changes_districts_change; Type: INDEX; Schema: ra; Owner: - -- -CREATE UNIQUE INDEX "index_ra.region_codes_on_code" ON region_codes USING btree (code); +CREATE INDEX street_name_changes_districts_change ON street_name_changes_districts USING btree (street_name_change_id); + + +-- +-- Name: street_name_changes_districts_district; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX street_name_changes_districts_district ON street_name_changes_districts USING btree (district_id); + + +-- +-- Name: street_name_changes_municipalities_change; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX street_name_changes_municipalities_change ON street_name_changes_municipalities USING btree (street_name_change_id); + + +-- +-- Name: street_name_changes_municipalities_municipality; Type: INDEX; Schema: ra; Owner: - +-- + +CREATE INDEX street_name_changes_municipalities_municipality ON street_name_changes_municipalities USING btree (municipality_id); SET search_path = rpo, pg_catalog; @@ -5116,6 +5683,22 @@ CREATE INDEX "index_socpoist.debtors_on_name" ON debtors USING btree (name); CREATE UNIQUE INDEX "index_socpoist.debts_lists_on_published_on" ON debts_lists USING btree (published_on); +SET search_path = upvs, pg_catalog; + +-- +-- Name: index_upvs.public_authority_edesks_on_cin; Type: INDEX; Schema: upvs; Owner: - +-- + +CREATE INDEX "index_upvs.public_authority_edesks_on_cin" ON public_authority_edesks USING btree (cin); + + +-- +-- Name: index_upvs.public_authority_edesks_on_uri; Type: INDEX; Schema: upvs; Owner: - +-- + +CREATE UNIQUE INDEX "index_upvs.public_authority_edesks_on_uri" ON public_authority_edesks USING btree (uri); + + SET search_path = vvo, pg_catalog; -- @@ -5377,6 +5960,280 @@ ALTER TABLE ONLY konkurz_restrukturalizacia_issues ADD CONSTRAINT fk_rails_fccf2c79e3 FOREIGN KEY (bulletin_issue_id) REFERENCES bulletin_issues(id) ON DELETE CASCADE; +SET search_path = ra, pg_catalog; + +-- +-- Name: fk_rails_0a2228fc64; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_0a2228fc64 FOREIGN KEY (municipality_id) REFERENCES municipalities(id); + + +-- +-- Name: fk_rails_0b4cfd5ea8; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipality_changes + ADD CONSTRAINT fk_rails_0b4cfd5ea8 FOREIGN KEY (municipality_id) REFERENCES municipalities(id); + + +-- +-- Name: fk_rails_13afcc56a3; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY region_changes + ADD CONSTRAINT fk_rails_13afcc56a3 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_1aa26b7b71; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_number_changes + ADD CONSTRAINT fk_rails_1aa26b7b71 FOREIGN KEY (building_number_id) REFERENCES building_numbers(id); + + +-- +-- Name: fk_rails_1be03a56b7; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipality_changes + ADD CONSTRAINT fk_rails_1be03a56b7 FOREIGN KEY (municipality_code_id) REFERENCES municipality_codes(id); + + +-- +-- Name: fk_rails_1dbeb0c531; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY region_changes + ADD CONSTRAINT fk_rails_1dbeb0c531 FOREIGN KEY (region_id) REFERENCES regions(id); + + +-- +-- Name: fk_rails_2812a036d9; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY district_changes + ADD CONSTRAINT fk_rails_2812a036d9 FOREIGN KEY (district_id) REFERENCES districts(id); + + +-- +-- Name: fk_rails_2e922593e0; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_2e922593e0 FOREIGN KEY (building_type_code_id) REFERENCES building_type_codes(id); + + +-- +-- Name: fk_rails_2f49a62832; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY county_changes + ADD CONSTRAINT fk_rails_2f49a62832 FOREIGN KEY (county_code_id) REFERENCES county_codes(id); + + +-- +-- Name: fk_rails_34207d0abe; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipality_changes + ADD CONSTRAINT fk_rails_34207d0abe FOREIGN KEY (county_id) REFERENCES counties(id); + + +-- +-- Name: fk_rails_36e1593362; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY county_changes + ADD CONSTRAINT fk_rails_36e1593362 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_3735b7a878; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_number_changes + ADD CONSTRAINT fk_rails_3735b7a878 FOREIGN KEY (street_name_id) REFERENCES street_names(id); + + +-- +-- Name: fk_rails_37c773e1c4; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY county_changes + ADD CONSTRAINT fk_rails_37c773e1c4 FOREIGN KEY (region_id) REFERENCES regions(id); + + +-- +-- Name: fk_rails_4eeaebe090; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY district_changes + ADD CONSTRAINT fk_rails_4eeaebe090 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_5651aa61bd; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_5651aa61bd FOREIGN KEY (district_id) REFERENCES districts(id); + + +-- +-- Name: fk_rails_56a851935d; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes_municipalities + ADD CONSTRAINT fk_rails_56a851935d FOREIGN KEY (street_name_change_id) REFERENCES street_name_changes(id); + + +-- +-- Name: fk_rails_5dd55655b7; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY county_changes + ADD CONSTRAINT fk_rails_5dd55655b7 FOREIGN KEY (county_id) REFERENCES counties(id); + + +-- +-- Name: fk_rails_6904502285; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipality_changes + ADD CONSTRAINT fk_rails_6904502285 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_6d4037406a; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes_districts + ADD CONSTRAINT fk_rails_6d4037406a FOREIGN KEY (street_name_change_id) REFERENCES street_name_changes(id); + + +-- +-- Name: fk_rails_7887f156dc; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes + ADD CONSTRAINT fk_rails_7887f156dc FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_866a34befa; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY district_changes + ADD CONSTRAINT fk_rails_866a34befa FOREIGN KEY (district_code_id) REFERENCES district_codes(id); + + +-- +-- Name: fk_rails_86db50199a; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_number_changes + ADD CONSTRAINT fk_rails_86db50199a FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_99d3af8f82; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_99d3af8f82 FOREIGN KEY (building_purpose_code_id) REFERENCES building_purpose_codes(id); + + +-- +-- Name: fk_rails_a3b287c1bd; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY municipality_changes + ADD CONSTRAINT fk_rails_a3b287c1bd FOREIGN KEY (city_id) REFERENCES municipalities(id); + + +-- +-- Name: fk_rails_a934dd39c9; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_unit_changes + ADD CONSTRAINT fk_rails_a934dd39c9 FOREIGN KEY (building_unit_id) REFERENCES building_units(id); + + +-- +-- Name: fk_rails_aeeb7b3888; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_aeeb7b3888 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_b65379a1e3; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY property_registration_number_changes + ADD CONSTRAINT fk_rails_b65379a1e3 FOREIGN KEY (property_registration_number_id) REFERENCES property_registration_numbers(id); + + +-- +-- Name: fk_rails_ba938ad380; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY district_changes + ADD CONSTRAINT fk_rails_ba938ad380 FOREIGN KEY (municipality_id) REFERENCES municipalities(id); + + +-- +-- Name: fk_rails_bf9792dcda; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY region_changes + ADD CONSTRAINT fk_rails_bf9792dcda FOREIGN KEY (region_code_id) REFERENCES region_codes(id); + + +-- +-- Name: fk_rails_ce071ee646; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_number_changes + ADD CONSTRAINT fk_rails_ce071ee646 FOREIGN KEY (property_registration_number_id) REFERENCES property_registration_numbers(id); + + +-- +-- Name: fk_rails_cecd401cf4; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes_districts + ADD CONSTRAINT fk_rails_cecd401cf4 FOREIGN KEY (district_id) REFERENCES districts(id); + + +-- +-- Name: fk_rails_d08f531de9; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY building_unit_changes + ADD CONSTRAINT fk_rails_d08f531de9 FOREIGN KEY (change_id) REFERENCES changes(id); + + +-- +-- Name: fk_rails_d12c0ce2ae; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes + ADD CONSTRAINT fk_rails_d12c0ce2ae FOREIGN KEY (street_name_id) REFERENCES street_names(id); + + +-- +-- Name: fk_rails_e8ff605033; Type: FK CONSTRAINT; Schema: ra; Owner: - +-- + +ALTER TABLE ONLY street_name_changes_municipalities + ADD CONSTRAINT fk_rails_e8ff605033 FOREIGN KEY (municipality_id) REFERENCES municipalities(id); + + SET search_path = rpo, pg_catalog; -- @@ -5743,6 +6600,242 @@ ALTER TABLE ONLY raw_notices SET search_path TO "$user", public; -INSERT INTO schema_migrations (version) VALUES ('20160228204654'), ('20160304144454'), ('20160304151009'), ('20160304210816'), ('20160304211624'), ('20160304213306'), ('20160305001135'), ('20160305084653'), ('20160307215611'), ('20160308210325'), ('20160309115526'), ('20160309120612'), ('20160429141807'), ('20160506091443'), ('20160506115555'), ('20160506135302'), ('20160506141328'), ('20160506145648'), ('20160506201551'), ('20160506201735'), ('20160506204302'), ('20160509085025'), ('20160509091003'), ('20160509123118'), ('20160509133921'), ('20160509142808'), ('20160509144034'), ('20160509202636'), ('20160509210342'), ('20160509211550'), ('20160509213609'), ('20160510141343'), ('20160511143847'), ('20160511174234'), ('20160512140751'), ('20160512141004'), ('20160512142220'), ('20160512163333'), ('20160512205002'), ('20160512222238'), ('20160512223007'), ('20160513090321'), ('20160513093332'), ('20160513100718'), ('20160513102554'), ('20160513111743'), ('20160513115940'), ('20160513123028'), ('20160513144927'), ('20160515073133'), ('20160515073826'), ('20160515074954'), ('20160516152548'), ('20160516153747'), ('20160516194853'), ('20160517092857'), ('20160517101512'), ('20160517102123'), ('20160517103922'), ('20160517110722'), ('20160517111053'), ('20160517113734'), ('20160517115800'), ('20160517125158'), ('20160517211611'), ('20160519200530'), ('20160520141455'), ('20160520142114'), ('20160520151100'), ('20160523062919'), ('20160523111212'), ('20160523134144'), ('20160523144401'), ('20160523150519'), ('20160524145856'), ('20160524155401'), ('20160524192650'), ('20160524192805'), ('20160524192859'), ('20160524195216'), ('20160524210510'), ('20160524210952'), ('20160524213930'), ('20160524221026'), ('20160524223055'), ('20160525105622'), ('20160525122831'), ('20160525202154'), ('20160525202912'), ('20160525203905'), ('20160603134420'), ('20160603134547'), ('20160603134923'), ('20160603201156'), ('20160603201256'), ('20160603202143'), ('20160604214151'), ('20160604222116'), ('20160605101834'), ('20160605192328'), ('20160605192524'), ('20160605193318'), ('20160605211444'), ('20160605211612'), ('20160607183913'), ('20160609161856'), ('20160609175034'), ('20160609183417'), ('20160610124301'), ('20160613145120'), ('20160613161922'), ('20160614082527'), ('20160615165505'), ('20160622105806'), ('20160622112911'), ('20160622113449'), ('20160622113649'), ('20160622114653'), ('20160622115927'), ('20160622142531'), ('20160622143425'), ('20160622144845'), ('20160622202723'), ('20160622231145'), ('20160623135959'), ('20160623150755'), ('20160624113525'), ('20160629144830'), ('20160629151520'), ('20160629201243'), ('20160629221305'), ('20160629221424'), ('20160629221800'), ('20160706111638'), ('20160707212141'), ('20160708105151'), ('20160708121934'), ('20160708140307'), ('20160708155010'), ('20160708172910'), ('20160708173417'), ('20160711125108'), ('20160711151815'), ('20160718092226'), ('20160718092417'), ('20160718094712'), ('20160719123714'), ('20160720085630'), ('20160721091930'), ('20160721134515'), ('20160721144047'), ('20160721151057'), ('20160721153228'), ('20160721153817'), ('20160721160533'), ('20160721160901'), ('20160721162155'), ('20160722151335'), ('20160722151503'), ('20160727150803'), ('20160803124909'), ('20160803142910'), ('20160803205746'), ('20160808125148'), ('20160809092452'), ('20160809095256'), ('20160809115742'), ('20160809120030'), ('20160811100934'), ('20160811102028'), ('20160811103149'), ('20160811104830'), ('20160811111706'), ('20160811112312'), ('20160811121506'), ('20160811124755'), ('20160812133817'), ('20160812135204'), ('20160812135415'), ('20160812140655'), ('20160812142245'), ('20160812142416'), ('20160812202908'), ('20160812215714'), ('20160815130928'), ('20160818110211'), ('20160818110621'), ('20160818135739'), ('20160825110839'), ('20160825120623'), ('20160830120608'), ('20160830121920'), ('20160928120044'), ('20160928123138'), ('20160928213723'), ('20160929120951'), ('20160929121253'), ('20160929121400'), ('20160929124430'), ('20160929124817'), ('20160929125059'), ('20160929125508'), ('20160930085831'), ('20160930102048'), ('20161010145325'), ('20161010145425'), ('20161010205914'), ('20161010220918'), ('20161010221017'), ('20161010224003'), ('20161010224400'), ('20161010225250'), ('20161011210352'), ('20161011214757'), ('20161012110655'), ('20161012135758'), ('20161012150803'), ('20161012150927'), ('20161012195055'), ('20161012195151'), ('20161012210141'), ('20161012210225'), ('20161012212459'), ('20161012212552'), ('20161013060038'); - - +INSERT INTO "schema_migrations" (version) VALUES +('20160228204654'), +('20160304144454'), +('20160304151009'), +('20160304210816'), +('20160304211624'), +('20160304213306'), +('20160305001135'), +('20160305084653'), +('20160307215611'), +('20160308210325'), +('20160309115526'), +('20160309120612'), +('20160429141807'), +('20160506091443'), +('20160506115555'), +('20160506135302'), +('20160506141328'), +('20160506145648'), +('20160506201551'), +('20160506201735'), +('20160506204302'), +('20160509085025'), +('20160509091003'), +('20160509123118'), +('20160509133921'), +('20160509142808'), +('20160509144034'), +('20160509202636'), +('20160509210342'), +('20160509211550'), +('20160509213609'), +('20160510141343'), +('20160511143847'), +('20160511174234'), +('20160512140751'), +('20160512141004'), +('20160512142220'), +('20160512163333'), +('20160512205002'), +('20160512222238'), +('20160512223007'), +('20160513090321'), +('20160513093332'), +('20160513100718'), +('20160513102554'), +('20160513111743'), +('20160513115940'), +('20160513123028'), +('20160513144927'), +('20160515073133'), +('20160515073826'), +('20160515074954'), +('20160516152548'), +('20160516153747'), +('20160516194853'), +('20160517092857'), +('20160517101512'), +('20160517102123'), +('20160517103922'), +('20160517110722'), +('20160517111053'), +('20160517113734'), +('20160517115800'), +('20160517125158'), +('20160517211611'), +('20160519200530'), +('20160520141455'), +('20160520142114'), +('20160520151100'), +('20160523062919'), +('20160523111212'), +('20160523134144'), +('20160523144401'), +('20160523150519'), +('20160524145856'), +('20160524155401'), +('20160524192650'), +('20160524192805'), +('20160524192859'), +('20160524195216'), +('20160524210510'), +('20160524210952'), +('20160524213930'), +('20160524221026'), +('20160524223055'), +('20160525105622'), +('20160525122831'), +('20160525202154'), +('20160525202912'), +('20160525203905'), +('20160603134420'), +('20160603134547'), +('20160603134923'), +('20160603201156'), +('20160603201256'), +('20160603202143'), +('20160604214151'), +('20160604222116'), +('20160605101834'), +('20160605192328'), +('20160605192524'), +('20160605193318'), +('20160605211444'), +('20160605211612'), +('20160607183913'), +('20160609161856'), +('20160609175034'), +('20160609183417'), +('20160610124301'), +('20160613145120'), +('20160613161922'), +('20160614082527'), +('20160615165505'), +('20160622105806'), +('20160622112911'), +('20160622113449'), +('20160622113649'), +('20160622114653'), +('20160622115927'), +('20160622142531'), +('20160622143425'), +('20160622144845'), +('20160622202723'), +('20160622231145'), +('20160623135959'), +('20160623150755'), +('20160624113525'), +('20160629144830'), +('20160629151520'), +('20160629201243'), +('20160629221305'), +('20160629221424'), +('20160629221800'), +('20160706111638'), +('20160707212141'), +('20160708105151'), +('20160708121934'), +('20160708140307'), +('20160708155010'), +('20160708172910'), +('20160708173417'), +('20160711125108'), +('20160711151815'), +('20160718092226'), +('20160718092417'), +('20160718094712'), +('20160719123714'), +('20160720085630'), +('20160721091930'), +('20160721134515'), +('20160721144047'), +('20160721151057'), +('20160721153228'), +('20160721153817'), +('20160721160533'), +('20160721160901'), +('20160721162155'), +('20160722151335'), +('20160722151503'), +('20160727150803'), +('20160803124909'), +('20160803142910'), +('20160803205746'), +('20160808125148'), +('20160809092452'), +('20160809095256'), +('20160809115742'), +('20160809120030'), +('20160811100934'), +('20160811102028'), +('20160811103149'), +('20160811104830'), +('20160811111706'), +('20160811112312'), +('20160811121506'), +('20160811124755'), +('20160812133817'), +('20160812135204'), +('20160812135415'), +('20160812140655'), +('20160812142245'), +('20160812142416'), +('20160812202908'), +('20160812215714'), +('20160815130928'), +('20160818110211'), +('20160818110621'), +('20160818135739'), +('20160825110839'), +('20160825120623'), +('20160830120608'), +('20160830121920'), +('20160928120044'), +('20160928123138'), +('20160928213723'), +('20160929120951'), +('20160929121253'), +('20160929121400'), +('20160929124430'), +('20160929124817'), +('20160929125059'), +('20160929125508'), +('20160930085831'), +('20160930102048'), +('20161010145325'), +('20161010145425'), +('20161010205914'), +('20161010220918'), +('20161010221017'), +('20161010224003'), +('20161010224400'), +('20161010225250'), +('20161011210352'), +('20161011214757'), +('20161012110655'), +('20161012135758'), +('20161012150803'), +('20161012150927'), +('20161012195055'), +('20161012195151'), +('20161012210141'), +('20161012210225'), +('20161012212459'), +('20161012212552'), +('20161013060038'), +('20161018113906'), +('20161018141504'), +('20161018141555'), +('20170222131821'), +('20170223074140'), +('20170430173505'), +('20170430190843'), +('20170502120756'), +('20170503112646'), +('20170503203539'), +('20170506104022'), +('20170506190014'), +('20170513200748'); diff --git a/lib/harvester_utils/downloader.rb b/lib/harvester_utils/downloader.rb new file mode 100644 index 00000000..16ef4010 --- /dev/null +++ b/lib/harvester_utils/downloader.rb @@ -0,0 +1,34 @@ +require 'typhoeus' +require 'tempfile' + +module HarvesterUtils + class Downloader + + DownloadError = Class.new(StandardError) + + def self.url_exists?(url) + code = Typhoeus.head(url).code + if code == 200 + true + elsif code == 404 + false + else + raise DownloadError, "Given url: '#{url}' returned status code: #{code}" + end + end + + def self.download_file(url) + response = Typhoeus.get(url) + raise DownloadError, "Unexpected response code: #{response.code} for url: #{url}" if response.code != 200 + file = Tempfile.new(rand(1_000_000).to_s) + file.binmode + file.write(response.body) + file.close + file + end + + def self.get(url) + Typhoeus.get(url) + end + end +end diff --git a/lib/tasks/ra.rake b/lib/tasks/ra.rake index 431e7878..b6dc99b4 100644 --- a/lib/tasks/ra.rake +++ b/lib/tasks/ra.rake @@ -3,4 +3,9 @@ namespace :ra do task sync: :environment do Ra::SyncChangesBatchesJob.perform_async end + + desc "Import initial batch for address register" + task init: :environment do + Ra::FetchInitialBatchJob.perform_async + end end diff --git a/lib/tasks/upvs.rake b/lib/tasks/upvs.rake new file mode 100644 index 00000000..fb604f46 --- /dev/null +++ b/lib/tasks/upvs.rake @@ -0,0 +1,6 @@ +namespace :upvs do + desc 'Sync public authority edesks' + task 'public_authority_edesks:sync' => :environment do + Upvs::FetchPublicAuthorityEdesksListJob.perform_later + end +end diff --git a/spec/jobs/ra/fetch_changes_batch_job_spec.rb b/spec/jobs/ra/fetch_changes_batch_job_spec.rb index 52ae34db..aa23af86 100644 --- a/spec/jobs/ra/fetch_changes_batch_job_spec.rb +++ b/spec/jobs/ra/fetch_changes_batch_job_spec.rb @@ -1,6 +1,12 @@ require 'rails_helper' RSpec.describe Ra::FetchChangesBatchJob, type: :job do + def parse_local_time(time_string) + Time.use_zone('Europe/Bratislava') do + Time.zone.parse(time_string) + end + end + describe '#perform' do let(:downloader) { double } @@ -12,46 +18,61 @@ property_change = Ra::PropertyRegistrationNumberChange.first expect(property_change).to have_attributes( - id: 60195291, - changed_at: Time.parse('2016-06-03T14:17:33.723'), + change_id: 60195291, + changed_at: parse_local_time('2016-06-03T14:17:33.723'), database_operation: 'INSERT', property_registration_number_id: 585020, version_id: 5245180, created_reason: 'CORRECT', - valid_from: Time.parse('2016-06-03T14:17:33.65'), - valid_to: Time.parse('3000-12-31T23:59:59.0'), + valid_from: parse_local_time('2016-06-03T14:17:33.65'), + valid_to: parse_local_time('3000-12-31T23:59:59.0'), effective_on: Date.parse('1000-01-01'), property_registration_number: 39, building_contains_flats: true, - building_type_id: 10, municipality_id: 2889, - district_id: 4697 + district_id: 4697, + + change: Ra::Change.find(60195291), + property_registration_number_object: Ra::PropertyRegistrationNumber.find(585020), + municipality: Ra::Municipality.find(2889), + district: Ra::District.find(4697), ) - expect(property_change.building_purpose).to have_attributes( + expect(property_change.building_purpose_code).to have_attributes( code: 'residentialBuilding', name: 'Bytová budova', ) - expect(property_change.building_type).to have_attributes( - id: 10, + expect(property_change.building_type_code).to have_attributes( + code: '10', name: 'Rodinný dom', ) building_number_change = Ra::BuildingNumberChange.first expect(building_number_change).to have_attributes( - id: 60195314, - changed_at: Time.parse('2016-06-03T14:17:33.877'), + change_id: 60195314, + changed_at: parse_local_time('2016-06-03T14:17:33.877'), database_operation: 'UPDATE', building_number_id: 2125309, version_id: 5207552, created_reason: 'IMPORT', - valid_from: Time.parse('1000-01-01T00:00:00.0'), - valid_to: Time.parse('2016-06-03T14:17:33.65'), + valid_from: parse_local_time('1000-01-01T00:00:00.0'), + valid_to: parse_local_time('2016-06-03T14:17:33.65'), effective_on: Date.parse('1000-01-01'), building_index: '1745974A', postal_code: 7214, property_registration_number_id: 585020, + street_name_id: nil, + + change: Ra::Change.find(60195314), + building_number_object: Ra::BuildingNumber.find(2125309), + property_registration_number: Ra::PropertyRegistrationNumber.find(585020), + ) + + building_number_change_with_street_name = Ra::BuildingNumberChange.find_by!(street_name_id: 59189) + expect(building_number_change_with_street_name).to have_attributes( + street_name_id: 59189, + street_name: Ra::StreetName.find(59189) ) expect(building_number_change.address_point).to have_attributes( @@ -62,24 +83,26 @@ building_unit_change = Ra::BuildingUnitChange.first expect(building_unit_change).to have_attributes( - id: 60196464, + change_id: 60196464, building_unit_id: 4005683, building_number_id: 2734787, - changed_at: Time.parse('2016-06-03T14:18:13.279'), + changed_at: parse_local_time('2016-06-03T14:18:13.279'), database_operation: 'INSERT', version_id: 3087790, created_reason: 'CREATE', - valid_from: Time.parse('2016-06-03T14:18:13.222'), - valid_to: Time.parse('3000-12-31T23:59:59.0'), + valid_from: parse_local_time('2016-06-03T14:18:13.222'), + valid_to: parse_local_time('3000-12-31T23:59:59.0'), effective_on: Date.parse('2016-06-03'), building_unit_floor: '1', building_unit_number: '17', building_unit_label: 'byt', + + change: Ra::Change.find(60196464), ) expect(Ra::ChangesBatch.first).to have_attributes( id: 5362764, - generated_at: Time.parse('2016-10-06T02:35:21.615'), # TODO time zone + generated_at: parse_local_time('2016-10-06T02:35:21.615'), ) end @@ -89,18 +112,26 @@ subject.perform('https://data.gov.sk/dataset/de3dd18f-9124-4acb-ae00-705555332256/resource/c0cf0bd9-f6e7-4fdd-8d3c-311fa504feab/download/zmenovadavka1952939.xml', downloader: downloader) - expect(Ra::StreetNameChange.first).to have_attributes( - id: 19764348, + street_name_change = Ra::StreetNameChange.first + expect(street_name_change).to have_attributes( + change_id: 19764348, street_name_id: 39693, - changed_at: Time.parse('2016-04-14T11:35:51.872'), + changed_at: parse_local_time('2016-04-14T11:35:51.872'), database_operation: 'UPDATE', version_id: 3034621, created_reason: 'IMPORT', - valid_from: Time.parse('1981-01-26T00:00:00.0'), - valid_to: Time.parse('2016-04-14T11:35:51.821'), + valid_from: parse_local_time('1981-01-26T00:00:00.0'), + valid_to: parse_local_time('2016-04-14T11:35:51.821'), effective_on: Date.parse('1981-01-26'), street_name: 'Juraja Hronca', - municipality_id: 3000 + + change: Ra::Change.find(19764348), + municipalities: [Ra::Municipality.find(3000)], + districts: [], + ) + + expect(street_name_change.street_name_object).to have_attributes( + id: 39693, ) end @@ -112,12 +143,16 @@ region_change = Ra::RegionChange.first expect(region_change).to have_attributes( + change_id: nil, region_id: 1, database_operation: 'INSERT', version_id: 1, created_reason: 'CREATE', valid_to: Time.parse('2004-04-30T23:59:59+02:00'), effective_on: Date.parse('1996-07-24+02:00'), + + change: nil, + region_object: Ra::Region.find(1), ) expect(region_change.region_code).to have_attributes( @@ -134,6 +169,7 @@ county_change = Ra::CountyChange.first expect(county_change).to have_attributes( + change_id: nil, county_id: 11, region_id: 1, database_operation: 'INSERT', @@ -141,6 +177,10 @@ created_reason: 'CREATE', valid_to: Time.parse('2004-04-30T23:59:59+02:00'), effective_on: Date.parse('1996-07-24+02:00'), + + change: nil, + county_object: Ra::County.find(11), + region: Ra::Region.find(1), ) expect(county_change.county_code).to have_attributes( @@ -157,17 +197,23 @@ municipality_change = Ra::MunicipalityChange.first expect(municipality_change).to have_attributes( - municipality_id: 1, + municipality_id: 92, county_id: 11, database_operation: 'INSERT', version_id: 1, created_reason: 'CREATE', valid_to: Time.parse('2004-04-30T23:59:59+02:00'), - effective_on: Date.parse('1996-07-24+02:00'), - municipality_status: 'MUNICIPALITY' + municipality_status: 'MUNICIPALITY', + + municipality_object: Ra::Municipality.find(92), + county: Ra::County.find(11), ) + # Attributes we don't have testing data for + expect(municipality_change).to respond_to(:change_id, :city_id, :change) + expect(municipality_change.municipality_code).to have_attributes( + code: '100000', name: 'Neznáma' ) end @@ -180,13 +226,18 @@ district_change = Ra::DistrictChange.first expect(district_change).to have_attributes( + change_id: nil, district_id: 4764, municipality_id: 92, database_operation: 'INSERT', version_id: 1, created_reason: 'CREATE', valid_to: Time.parse('2004-04-30T23:59:59+02:00'), - unique_numbering: false + unique_numbering: false, + + change: nil, + district_object: Ra::District.find(4764), + municipality: Ra::Municipality.find(92), ) expect(district_change.district_code).to have_attributes( @@ -195,7 +246,15 @@ ) end - pending 'it adds batch record last' + it 'creates batch record only after successful import' do + expect(downloader).to receive(:download_file).with('abc'). + and_return(fixture_filepath('jobs/ra/fixtures/zmenovadavka1952939_with_invalid_tag.xml')) + + to_not_change_changes_batch_count = change { Ra::ChangesBatch.count }.by(0) + expect { subject.perform('abc', downloader: downloader) }.to( + raise_error(RuntimeError).and to_not_change_changes_batch_count + ) + end end end diff --git a/spec/jobs/ra/fixtures/zmenovadavka1952939_with_invalid_tag.xml b/spec/jobs/ra/fixtures/zmenovadavka1952939_with_invalid_tag.xml new file mode 100644 index 00000000..617ea798 --- /dev/null +++ b/spec/jobs/ra/fixtures/zmenovadavka1952939_with_invalid_tag.xml @@ -0,0 +1,7 @@ + + + RA-Core-TID-eac764c9-879b-4aab-9dec-4a3787233db3119529392016-05-02T02:20:45.349STREET_NAME7197643482016-04-14T11:35:51.872UPDATE396933034621IMPORT1981-01-26T00:00:00.02016-04-14T11:35:51.8211981-01-26Juraja Hronca3000 + + + + diff --git a/spec/jobs/ra/sync_changes_batches_job_spec.rb b/spec/jobs/ra/sync_changes_batches_job_spec.rb index a33af6d8..669afcf9 100644 --- a/spec/jobs/ra/sync_changes_batches_job_spec.rb +++ b/spec/jobs/ra/sync_changes_batches_job_spec.rb @@ -12,9 +12,24 @@ subject.perform(downloader: downloader, job: job) expect(job).to have_received(:perform_async).with('https://data.gov.sk/dataset/de3dd18f-9124-4acb-ae00-705555332256/resource/be6e0033-8004-41d7-8929-0484e24e9347/download/zmenovadavka234191.xml') - expect(job).to have_received(:perform_async).with('https://data.gov.sk/dataset/de3dd18f-9124-4acb-ae00-705555332256/resource/a2f3260d-1647-4953-888d-26688e07dca2/download/zmenovadavka5369741.xml') + expect(job).to have_received(:perform_async).with('https://data.gov.sk/dataset/de3dd18f-9124-4acb-ae00-705555332256/resource/f04158cc-9f94-4a96-8165-0aace413c9ab/download/zmenovadavka1952939.xml') end - pending 'schedules download of missing changes batches only' + it 'schedules download of missing changes batches only' do + expect(downloader).to receive(:get).with('https://data.gov.sk/dataset/register-adries-zmenove-davky'). + and_return(fake_response('jobs/ra/fixtures/changes-datasets.html')) + + Ra::FetchChangesBatchJob.new.perform_on_file(fixture_filepath('jobs/ra/fixtures/zmenovadavka1952939.xml')) + subject.perform(downloader: downloader, job: job) + + expect(job).to_not have_received(:perform_async).with(/zmenovadavka1952939.xml/) + end + + it 'fails if it receives a non-200 response' do + expect(downloader).to receive(:get). + and_return(fake_response('jobs/ra/fixtures/changes-datasets.html', status: 504)) + + expect{ subject.perform(downloader: downloader, job: job) }.to raise_error(RuntimeError) + end end end diff --git a/spec/jobs/upvs/fetch_public_authority_edesks_list_job_spec.rb b/spec/jobs/upvs/fetch_public_authority_edesks_list_job_spec.rb new file mode 100644 index 00000000..544d5d18 --- /dev/null +++ b/spec/jobs/upvs/fetch_public_authority_edesks_list_job_spec.rb @@ -0,0 +1,24 @@ +require 'rails_helper' + +RSpec.describe Upvs::FetchPublicAuthorityEdesksListJob, type: :job do + context '#perform' do + it 'downloads and imports PublicAuthorityEdesks from data.gov.sk' do + downloader = double + expect(downloader).to receive(:download_file).with('https://data.gov.sk/dataset/7d918796-3c9a-44a5-95a9-7e1dfca76407/resource/456a5bc6-32bd-4346-bcd1-12ee171ac010/download/zoznamintituciisozriadenoues.csv').and_return(fixture_filepath('jobs/upvs/fixtures/upvs-edesks.csv')) + + subject.perform(downloader: downloader) + + expect(Upvs::PublicAuthorityEdesk.first).to have_attributes( + cin: 42156424, + name: 'Národná agentúra pre sieťové a elektronické služby: Test', + street: 'Kollárova', + street_number: nil, + postal_code: '91702', + city: 'Trnava', + uri: 'ico://sk/42156424_90000' + ) + + expect(Upvs::PublicAuthorityEdesk.count).to eq(5) + end + end +end diff --git a/spec/jobs/upvs/fixtures/upvs-edesks.csv b/spec/jobs/upvs/fixtures/upvs-edesks.csv new file mode 100644 index 00000000..ece5f0a2 --- /dev/null +++ b/spec/jobs/upvs/fixtures/upvs-edesks.csv @@ -0,0 +1,6 @@ +_id,IČO,NAZOV INŠTITÚCIE,ULICA,ČÍSLO,PSČ,MESTO,URI,Dátum poslednej aktualizácie: 9.12.2016 +1,42156424,Národná agentúra pre sieťové a elektronické služby: Test,Kollárova,,91702,Trnava,ico://sk/42156424_90000, +2,309591,Obec Jablonka,,51,90621,Jablonka,ico://sk/00309591, +3,309338,Obec Tvrdošovce,,,,Tvrdošovce,ico://sk/00309338, +4,310506,Obec Chynorany,,,,Chynorany,ico://sk/00310506, +5,166260,"Úrad geodézie, kartografie a katastra Slovenskej republiky",Ružová dolina,27,82109,Bratislava - mestská časť Ružinov,ico://sk/00166260, diff --git a/spec/models/ra/building_number_change_spec.rb b/spec/models/ra/building_number_change_spec.rb deleted file mode 100644 index ce6e2aac..00000000 --- a/spec/models/ra/building_number_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::BuildingNumberChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/building_purpose_spec.rb b/spec/models/ra/building_purpose_spec.rb deleted file mode 100644 index a8e198d6..00000000 --- a/spec/models/ra/building_purpose_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::BuildingPurpose, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/building_type_spec.rb b/spec/models/ra/building_type_spec.rb deleted file mode 100644 index d1550010..00000000 --- a/spec/models/ra/building_type_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::BuildingType, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/building_unit_change_spec.rb b/spec/models/ra/building_unit_change_spec.rb deleted file mode 100644 index f0eeeb70..00000000 --- a/spec/models/ra/building_unit_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::BuildingUnitChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/changes_batch_spec.rb b/spec/models/ra/changes_batch_spec.rb deleted file mode 100644 index 15a853eb..00000000 --- a/spec/models/ra/changes_batch_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::ChangesBatch, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/county_change_spec.rb b/spec/models/ra/county_change_spec.rb deleted file mode 100644 index b93c9d50..00000000 --- a/spec/models/ra/county_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::CountyChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/county_code_spec.rb b/spec/models/ra/county_code_spec.rb deleted file mode 100644 index 441eef03..00000000 --- a/spec/models/ra/county_code_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::CountyCode, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/district_change_spec.rb b/spec/models/ra/district_change_spec.rb deleted file mode 100644 index 0f2399af..00000000 --- a/spec/models/ra/district_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::DistrictChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/district_code_spec.rb b/spec/models/ra/district_code_spec.rb deleted file mode 100644 index 70aff0a8..00000000 --- a/spec/models/ra/district_code_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::DistrictCode, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/municipality_change_spec.rb b/spec/models/ra/municipality_change_spec.rb deleted file mode 100644 index 941c9cc8..00000000 --- a/spec/models/ra/municipality_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::MunicipalityChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/municipality_code_spec.rb b/spec/models/ra/municipality_code_spec.rb deleted file mode 100644 index 4c6f297a..00000000 --- a/spec/models/ra/municipality_code_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::MunicipalityCode, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/property_registration_number_change_spec.rb b/spec/models/ra/property_registration_number_change_spec.rb deleted file mode 100644 index 7bd8259e..00000000 --- a/spec/models/ra/property_registration_number_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::PropertyRegistrationNumberChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/region_change_spec.rb b/spec/models/ra/region_change_spec.rb deleted file mode 100644 index 7240f898..00000000 --- a/spec/models/ra/region_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::RegionChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/region_code_spec.rb b/spec/models/ra/region_code_spec.rb deleted file mode 100644 index d3f11fca..00000000 --- a/spec/models/ra/region_code_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::RegionCode, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/ra/street_name_change_spec.rb b/spec/models/ra/street_name_change_spec.rb deleted file mode 100644 index 6b0857bb..00000000 --- a/spec/models/ra/street_name_change_spec.rb +++ /dev/null @@ -1,5 +0,0 @@ -require 'rails_helper' - -RSpec.describe Ra::StreetNameChange, type: :model do - pending "add some examples to (or delete) #{__FILE__}" -end diff --git a/spec/models/upvs/public_authority_edesk_spec.rb b/spec/models/upvs/public_authority_edesk_spec.rb new file mode 100644 index 00000000..89bd8794 --- /dev/null +++ b/spec/models/upvs/public_authority_edesk_spec.rb @@ -0,0 +1,4 @@ +require 'rails_helper' + +RSpec.describe Upvs::PublicAuthorityEdesk, type: :model do +end diff --git a/spec/rails_helper.rb b/spec/rails_helper.rb index 43834296..b9b9fd8a 100644 --- a/spec/rails_helper.rb +++ b/spec/rails_helper.rb @@ -6,6 +6,7 @@ require 'spec_helper' require 'rspec/rails' # Add additional requires below this line. Rails is not loaded until this point! +Dir[Rails.root.join('spec/support/**/*.rb')].sort.each { |f| require f } # Requires supporting ruby files with custom matchers and macros, etc, in # spec/support/ and its subdirectories. Files matching `spec/**/*_spec.rb` are diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 8f698be4..70ab7910 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -16,6 +16,9 @@ # users commonly want. # # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +require 'simplecov' +SimpleCov.start + RSpec.configure do |config| # rspec-expectations config goes here. You can use an alternate # assertion/expectation library such as wrong or the stdlib/minitest diff --git a/spec/support/fixture_macros.rb b/spec/support/fixture_macros.rb new file mode 100644 index 00000000..c888aeee --- /dev/null +++ b/spec/support/fixture_macros.rb @@ -0,0 +1,13 @@ +module FixtureMacros + def fixture_filepath(file_path) + Pathname.new('spec').join(file_path) + end + + def file_fixture(file_path) + File.read(fixture_filepath(file_path)) + end +end + +RSpec.configure do |config| + config.include FixtureMacros +end