From dcfeb49b222e633b738def1fab9eba12ed08d40f Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:04:30 +0000 Subject: [PATCH 01/27] Fix bugged migration --- db/migrate/20180112204406_startup.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20180112204406_startup.rb b/db/migrate/20180112204406_startup.rb index 18322e2..5206d23 100644 --- a/db/migrate/20180112204406_startup.rb +++ b/db/migrate/20180112204406_startup.rb @@ -13,7 +13,7 @@ def change alter table migrate_cache drop key full; alter table nerve_cache drop key `external_id`, drop key `n_uuid`, drop key `n_tr`, drop key `n_td`; - alter table tracks drop key in_artist, drop key in_album, drop key in_title, drop key index_title, drop key external_id; + alter table tracks drop key in_artist, drop key in_album, drop key in_title, drop key index_title; alter table users drop key external_ref; From 85454659adf352b364a257ae601f5565b8f42bf2 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:10:37 +0000 Subject: [PATCH 02/27] Move migrations into independent statements --- db/migrate/20180112204406_startup.rb | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/db/migrate/20180112204406_startup.rb b/db/migrate/20180112204406_startup.rb index 5206d23..f826d84 100644 --- a/db/migrate/20180112204406_startup.rb +++ b/db/migrate/20180112204406_startup.rb @@ -1,23 +1,23 @@ class Startup < ActiveRecord::Migration def change - execute <<-SQL + begin + execute "alter table albums drop key `in_artist`;" + + execute "alter table albums drop key `external_id`;" - alter table albums drop key `in_artist`; - alter table albums drop key `external_id`; + execute "alter table artists drop key `external_id`;" + execute "alter table artists drop key in_artist;" + execute "alter table artists drop key index_artist;" - alter table artists drop key `external_id`; - alter table artists drop key in_artist; - alter table artists drop key index_artist; + execute "alter table migrate_cache drop key full;" + execute "alter table nerve_cache drop key `external_id`, drop key `n_uuid`, drop key `n_tr`, drop key `n_td`;" - alter table migrate_cache drop key full; - alter table nerve_cache drop key `external_id`, drop key `n_uuid`, drop key `n_tr`, drop key `n_td`; + execute "alter table tracks drop key in_artist, drop key in_album, drop key in_title, drop key index_title;" - alter table tracks drop key in_artist, drop key in_album, drop key in_title, drop key index_title; - - alter table users drop key external_ref; - - SQL + execute "alter table users drop key external_ref;" + rescue + end end end From f5e6842319930862f354cfc52de998830a400812 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:16:42 +0000 Subject: [PATCH 03/27] Use initialize_by instead of create_by to avoid default key error --- jobs/process.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jobs/process.rb b/jobs/process.rb index bb66674..7e5b93f 100644 --- a/jobs/process.rb +++ b/jobs/process.rb @@ -282,12 +282,12 @@ def save options # Create the artist and album rows, if they don't already exist. - artist = Nerve::Model::Artist.find_or_create_by(name: data['artist']) + artist = Nerve::Model::Artist.find_or_initialize_by(name: data['artist']) artist.created_by = options['user_id'] #artist.external_id = ext_artist_id if ext_artist_id != nil artist.save! - album = Nerve::Model::Album.find_or_create_by(name: data['album'], artist: artist) + album = Nerve::Model::Album.find_or_initialize_by(name: data['album'], artist: artist) album.created_by = options['user_id'] #album.external_id = ext_album_id if ext_album_id != nil album.save! From 4fd55bb1ce07e5baff5ecbc1a1566eab4cb4d9e6 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:22:12 +0000 Subject: [PATCH 04/27] Make permissions serialized --- app/models/user.rb | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 7b5de9b..de76b7c 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -13,11 +13,11 @@ class User < ActiveRecord::Base alias_attribute :moderator, :is_moderator alias_attribute :playout_username, :playout - attr_reader :permissions + serialize :permissions def after_initialize - @permissions = { + @_permissions = { :override_bitrate => false, :override_compressor => false, :upload_library => false, @@ -27,18 +27,18 @@ def after_initialize :safety_net => true, } - permissions.each { | k, v| @permissions[k] = v } + permissions.each { | k, v| @_permissions[k] = v } if admin or moderator - @permissions[:override_bitrate] = true - @permissions[:override_compressor] = true - @permissions[:upload_library] = true - @permissions[:instrumental] = true - @permissions[:upload_automation] = true - @permissions[:safety_net] = false + @_permissions[:override_bitrate] = true + @_permissions[:override_compressor] = true + @_permissions[:upload_library] = true + @_permissions[:instrumental] = true + @_permissions[:upload_automation] = true + @_permissions[:safety_net] = false end - permissions = @permissions + permissions = @_permissions end @@ -64,4 +64,4 @@ def to_json state = nil end -end; end \ No newline at end of file +end; end From a48a251a6cde6b72af84af838d9ac6ea47773a03 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:24:23 +0000 Subject: [PATCH 05/27] Serialize with JSON --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/user.rb b/app/models/user.rb index de76b7c..3fbc9c5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -13,7 +13,7 @@ class User < ActiveRecord::Base alias_attribute :moderator, :is_moderator alias_attribute :playout_username, :playout - serialize :permissions + serialize :permissions, JSON def after_initialize From 524c30ba8530d599b6a39ed8b43d8fde93e9ba27 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:26:06 +0000 Subject: [PATCH 06/27] Bugfix: auth not setting right permissions --- auth/cortex.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth/cortex.rb b/auth/cortex.rb index 079d1bf..5524fe2 100644 --- a/auth/cortex.rb +++ b/auth/cortex.rb @@ -68,7 +68,7 @@ def authorize session, params, request user.admin = !session[:user]["groups"][admin].nil? - if !session[:user]["groups"][specialist].nil? + if !session[:user]["groups"][specialist].nil? or user.admin or user.moderator user.permissions[:override_bitrate] = true user.permissions[:override_compressor] = true user.permissions[:instrumental] = true From 949d1589aa8cf6ff9de40b6e373b795c713f367d Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:28:07 +0000 Subject: [PATCH 07/27] Add after_initialize for user --- app/models/user.rb | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/models/user.rb b/app/models/user.rb index 3fbc9c5..3cbc9c5 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -62,6 +62,10 @@ def to_json state = nil get_json.to_json end + after_initialize do | user | + user.after_initialize + end + end end; end From 2176cf94d86cc2e7c0e20846221af471bb9e0b36 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:40:29 +0000 Subject: [PATCH 08/27] Use string permissions instead of symbols --- app/models/user.rb | 29 ++++++++++++++--------------- auth/cortex.rb | 10 +++++----- services/upload.rb | 4 ++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 3cbc9c5..97618cf 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -18,24 +18,24 @@ class User < ActiveRecord::Base def after_initialize @_permissions = { - :override_bitrate => false, - :override_compressor => false, - :upload_library => false, - :upload_automation => false, - :instrumental => false, + 'override_bitrate' => false, + 'override_compressor' => false, + 'upload_library' => false, + 'upload_automation' => false, + 'instrumental' => false, # Safety net ensures the first 3 uploads for a user are moderated. TODO - :safety_net => true, + 'safety_net' => true, } permissions.each { | k, v| @_permissions[k] = v } if admin or moderator - @_permissions[:override_bitrate] = true - @_permissions[:override_compressor] = true - @_permissions[:upload_library] = true - @_permissions[:instrumental] = true - @_permissions[:upload_automation] = true - @_permissions[:safety_net] = false + @_permissions['override_bitrate'] = true + @_permissions['override_compressor'] = true + @_permissions['upload_library'] = true + @_permissions['instrumental'] = true + @_permissions['upload_automation'] = true + @_permissions['safety_net'] = false end permissions = @_permissions @@ -62,9 +62,8 @@ def to_json state = nil get_json.to_json end - after_initialize do | user | - user.after_initialize - end + after_initialize :after_initialize + after_load :after_initialize end diff --git a/auth/cortex.rb b/auth/cortex.rb index 5524fe2..9cfb604 100644 --- a/auth/cortex.rb +++ b/auth/cortex.rb @@ -69,12 +69,12 @@ def authorize session, params, request if !session[:user]["groups"][specialist].nil? or user.admin or user.moderator - user.permissions[:override_bitrate] = true - user.permissions[:override_compressor] = true - user.permissions[:instrumental] = true + user.permissions['override_bitrate'] = true + user.permissions['override_compressor'] = true + user.permissions['instrumental'] = true else - user.permissions[:override_bitrate] = false - user.permissions[:override_compressor] = false + user.permissions['override_bitrate'] = false + user.permissions['override_compressor'] = false end user.save! diff --git a/services/upload.rb b/services/upload.rb index 81fe534..5fdb4a6 100644 --- a/services/upload.rb +++ b/services/upload.rb @@ -55,7 +55,7 @@ def self.get_total_uploads login_service = Nerve::Services::Login.get_service @user = login_service.get_user(session[:user_id]) rescue login_service.get_nil_user - instrumental = (params['instrumental'] == "true" and @user.permissions[:instrumental]) + instrumental = (params['instrumental'] == "true" and @user.permissions['instrumental']) data = { "file" => temp_file, "user_id" => session[:user_id], @@ -65,7 +65,7 @@ def self.get_total_uploads "album" => params['album'], "override_bitrate" => ( params['override_bitrate'] == "true" and - @user.permissions[:override_bitrate]), # TODO: check user can do that + @user.permissions['override_bitrate']), # TODO: check user can do that "override_compressor" => params['override_compressor'] == "true", "upload_library" => params['upload_library'] == "true", "instrumental" => instrumental From a453783ad0d86278e9adecdf92cef84a09d97902 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:42:19 +0000 Subject: [PATCH 09/27] Whoops! --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/user.rb b/app/models/user.rb index 97618cf..8b63bab 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -63,7 +63,7 @@ def to_json state = nil end after_initialize :after_initialize - after_load :after_initialize + after_find :after_initialize end From 48d10b9bbbf87c8791bd2cc74d039527e610fe3c Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:47:13 +0000 Subject: [PATCH 10/27] Force update attribute hack --- app/models/user.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/user.rb b/app/models/user.rb index 8b63bab..78d0f32 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -38,7 +38,7 @@ def after_initialize @_permissions['safety_net'] = false end - permissions = @_permissions + write_attribute :permissions, @_permissions end From f8ac734a9d1c95b72f0e960078bb7ccf2941d6e8 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:51:25 +0000 Subject: [PATCH 11/27] Update AudioWall and such to use new track API --- app/mixin/spotify.rb | 4 ++-- playout/audiowall.rb | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/mixin/spotify.rb b/app/mixin/spotify.rb index d19feb6..14300b9 100644 --- a/app/mixin/spotify.rb +++ b/app/mixin/spotify.rb @@ -25,7 +25,7 @@ def self.connect_to_api def self.search track, token title_clean = track.title.split(/(ft\.|feat\.?\s)/)[0].gsub('"', '').gsub(/[\(\[][a-zA-Z0-9\ \.\-]+[\)\]]/, '').split("feat.")[0].strip - artist_clean = track.artist.split(/(ft\.|feat\.?\s)/)[0].gsub('"', '').split("feat.")[0].strip + artist_clean = track.artist.name.split(/(ft\.|feat\.?\s)/)[0].gsub('"', '').split("feat.")[0].strip http_response = RestClient.get("https://api.spotify.com/v1/search", {params: {q: 'track:"' + title_clean + '" artist:"' + artist_clean + '"', type: "track", market: "GB"}, authorization: token}) response = JSON.parse(http_response.body) rescue nil @@ -111,4 +111,4 @@ def self.pre_publish track, playout end -end; end \ No newline at end of file +end; end diff --git a/playout/audiowall.rb b/playout/audiowall.rb index 5330613..f4c4cc1 100644 --- a/playout/audiowall.rb +++ b/playout/audiowall.rb @@ -28,7 +28,7 @@ def initialize track = nil return unless track @track = track - @title = track.title; @artist = track.artist; @description = ""; + @title = track.title; @artist = track.artist.name; @description = ""; @hook_start = track.hook_start; @hook_end = track.hook_end @intro_start = track.intro_start; @intro_end = track.intro_end @extro_start = track.outro; @extro_end = track.length From f492789351403272bee5463081f77497d8d7d0cf Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:53:01 +0000 Subject: [PATCH 12/27] Fix AW --- playout/audiowall/database.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/playout/audiowall/database.rb b/playout/audiowall/database.rb index e41b240..1489f7e 100644 --- a/playout/audiowall/database.rb +++ b/playout/audiowall/database.rb @@ -162,7 +162,7 @@ def insert_into_deck item_number, category_id def add_track cart_id, track, category_id, alt_category_id = 0, options = {} - artist_id = get_artist_id track.artist + artist_id = get_artist_id track.artist.name title_id = get_title_id track.title style_id = get_style track.genre, $genres[track.genre] @@ -227,7 +227,7 @@ def add_track cart_id, track, category_id, alt_category_id = 0, options = {} def update_track track, options - artist_id = get_artist_id track.artist + artist_id = get_artist_id track.artist.name title_id = get_title_id track.title arguments = [] @@ -288,4 +288,4 @@ def get_cart track end -end; end; end \ No newline at end of file +end; end; end From 0c7487acafbd9b61ac7c12dfd60cb1d1f6daf7cb Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 10:54:29 +0000 Subject: [PATCH 13/27] Fix AW --- playout/audiowall/database.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/playout/audiowall/database.rb b/playout/audiowall/database.rb index 1489f7e..4d6e96c 100644 --- a/playout/audiowall/database.rb +++ b/playout/audiowall/database.rb @@ -200,7 +200,7 @@ def add_track cart_id, track, category_id, alt_category_id = 0, options = {} '%s', GETDATE(), GETDATE(), 0, %i, '%s', '%s', '%s', '%s')", SecureRandom.uuid, nil, 'Nerve_' + track.id.to_s, - title_id, artist_id, nil, @conn.escape(track.title), @conn.escape(track.artist), + title_id, artist_id, nil, @conn.escape(track.title), @conn.escape(track.artist.name), cart_id, category_id, alt_category_id, 0, era_id, 0, (options['CharacteristicStart1'] or 0), (options['CharacteristicEnd1'] or 0), (options['CharacteristicStart2'] or 0), (options['CharacteristicEnd2'] or 0), @@ -234,7 +234,7 @@ def update_track track, options query = [] options['DisplayTitle'] = track.title - options['DisplayBy'] = track.artist + options['DisplayBy'] = track.artist.name options.each { | k, v | arguments << k + " = '%s'"; query << @conn.escape(v.to_s) } From 877129da1f2bb52e97d9d22384cc8fbd6df77c65 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 18:13:15 +0000 Subject: [PATCH 14/27] Replace title with track in cache method --- services/metadata.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/services/metadata.rb b/services/metadata.rb index f2ae251..2cf08f9 100644 --- a/services/metadata.rb +++ b/services/metadata.rb @@ -77,6 +77,7 @@ def self.cache data item = Nerve::Model::CacheItem.find_or_create_by(external_id: data['external_id']) data['year'] = data['year'] == '' ? 0 : data['year'] + data['track'] = data['title'] data = data.select {|x| item.class.attribute_names.index(x) } item.assign_attributes data From 40a53d22f036d3fef79f4ec05a93ab433c0368ec Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 20:51:44 +0000 Subject: [PATCH 15/27] Update Track model to avoid local variable overwrite, so delete methods work --- app/models/track.rb | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/app/models/track.rb b/app/models/track.rb index 552cec7..4be1d04 100644 --- a/app/models/track.rb +++ b/app/models/track.rb @@ -27,16 +27,16 @@ def approved def get_metadata result = Nerve::Services::Metadata.match_meta(external_id, true, "external_id") - big = result["big"] rescue nil + _big = result["big"] rescue nil - lyrics = JSON.parse(big["lyrics"])[0] rescue "" + _lyrics = JSON.parse(_big["lyrics"])[0] rescue "" - if !lyrics or lyrics.to_s.empty? - lyrics = Nerve::Services::Metadata.match_lyrics(artist.name, album.name, title)[0] \ + if !_lyrics or _lyrics.to_s.empty? + _lyrics = Nerve::Services::Metadata.match_lyrics(artist.name, album.name, title)[0] \ rescue "No lyrics available." end - [result, lyrics, (result["year"] rescue nil)] + [result, _lyrics, (result["year"] rescue nil)] end @@ -44,7 +44,7 @@ def get_metadata def is_safe return false if explicit or status == 0 - extended, lyrics = get_metadata + _extended, _lyrics = get_metadata return true if instrumental @@ -53,10 +53,10 @@ def is_safe r = Regexp.new("\\b((#{$config["words"]["banned"].join("|")})[^\\s\\b,.\<\>]*)\\b", 7) # If the size is 0, then it's "safe" ish - return false if !lyrics or lyrics.to_s.empty? + return false if !_lyrics or _lyrics.to_s.empty? # TODO: make this cleaner - lyrics.scan(r).size == 0 && !!lyrics && lyrics != "" && lyrics != "No lyrics available." + _lyrics.scan(r).size == 0 && !!_lyrics && _lyrics != "" && _lyrics != "No lyrics available." end @@ -68,27 +68,27 @@ def why_unsafe return "determined as risky during upload" if status == 0 # only works before submission return "has parental advisory/explicit flag" if explicit - extended, lyrics = get_metadata + _extended, _lyrics = get_metadata r = Regexp.new("\\b((#{$config["words"]["banned"].join("|")})[^\\s\\b,.\<\>]*)\\b", 7) - s = lyrics.scan(r) + s = _lyrics.scan(r) return "it contains (at least one) expletive (#{s[0]})" if s.size > 0 - return "no lyrics were found" if !lyrics or lyrics == "No lyrics available." + return "no lyrics were found" if !_lyrics or _lyrics == "No lyrics available." false end def delete! soft = false - local_path = $config["export"]["directory"] + "/" + local_path - File.unlink(local_path) rescue nil + _local_path = $config["export"]["directory"] + "/" + local_path + File.unlink(_local_path) rescue nil if soft self.status = 6 self.save else - File.unlink(local_path + ".ogg") rescue nil - File.unlink(local_path + ".dat") rescue nil + File.unlink(_local_path + ".ogg") rescue nil + File.unlink(_local_path + ".dat") rescue nil destroy end end From 34881c6fcb6a89ac6fd00e324030d4f264ae28b5 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 21:45:34 +0000 Subject: [PATCH 16/27] Set default extra va --- app/mixin/spotify.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/mixin/spotify.rb b/app/mixin/spotify.rb index 14300b9..870badc 100644 --- a/app/mixin/spotify.rb +++ b/app/mixin/spotify.rb @@ -54,6 +54,7 @@ def self.created track images = result['album']['images'] rescue [] images.sort! { | a, b | b['width'] <=> a['width'] } + track.extra ||= {} track.extra['spotify_id'] = spotify_id track.extra['album_art'] = images[0]['url'] if images.length and images[0]['width'] > 400 track.save From f5b12dfdacaba7b393a8f193b8abb89206b77bfd Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 11 Feb 2018 21:55:28 +0000 Subject: [PATCH 17/27] Fix that stupid fucking incorrect file path bug again --- jobs/process.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/process.rb b/jobs/process.rb index 7e5b93f..ae38522 100644 --- a/jobs/process.rb +++ b/jobs/process.rb @@ -189,7 +189,7 @@ def compress(input, output, format, options) File.rename(output, input) # And finally, get the new loudness data. We need it to work out gain. - ld = investigate_loudness(output) + ld = investigate_loudness(input) end From 91f6d6fcfabf79811bcb4b970467d3507a0be4ce Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Mon, 12 Feb 2018 11:35:16 +0000 Subject: [PATCH 18/27] Fix get_json with no user --- app/models/track.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/track.rb b/app/models/track.rb index 552cec7..c64aee1 100644 --- a/app/models/track.rb +++ b/app/models/track.rb @@ -114,7 +114,7 @@ def get_json extended = false "is_automation" => is_automation, "playout_id" => playout_id, "end_type" => end_type, - "created_by" => user.get_json, + "created_by" => user && user.get_json, "flagged" => flagged, "instrumental" => instrumental, "extra" => extra From 8bafd114de2599e531212e7b2bc4e76c4cee6568 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sat, 17 Feb 2018 11:40:15 +0000 Subject: [PATCH 19/27] Update backend metadata. Fix bug where tracks were always being manually approved --- app/models/cache_item.rb | 10 ++++ app/models/track.rb | 53 +++++++++++++------ .../20180217105112_update_track_cols.rb | 13 +++++ db/schema.rb | 12 ++--- services/metadata.rb | 14 ++--- services/upload.rb | 4 +- 6 files changed, 73 insertions(+), 33 deletions(-) create mode 100644 db/migrate/20180217105112_update_track_cols.rb diff --git a/app/models/cache_item.rb b/app/models/cache_item.rb index 899e29b..2304de5 100644 --- a/app/models/cache_item.rb +++ b/app/models/cache_item.rb @@ -18,6 +18,16 @@ def exists? Track.where('external_id=?', external_id).count > 0 end + def init_by data + + data['year'] = data['year'] == '' ? 0 : data['year'] + data['track'] = data['title'] + + data = data.select {|x| self.class.attribute_names.index(x) } + assign_attributes data + + end + def get_json enhanced = false return { "cache_id" => id, diff --git a/app/models/track.rb b/app/models/track.rb index 4be1d04..364aec1 100644 --- a/app/models/track.rb +++ b/app/models/track.rb @@ -24,16 +24,34 @@ def approved !!status and status > 3 end + def update_metadata + + result = Nerve::Services::Metadata.match artist.name, (album.name rescue ''), title, true, true + external_id = result['external_id'] + get_metadata + + end + def get_metadata - result = Nerve::Services::Metadata.match_meta(external_id, true, "external_id") + result, cache_item = Nerve::Services::Metadata.match_meta(external_id, true, "external_id", true) _big = result["big"] rescue nil - _lyrics = JSON.parse(_big["lyrics"])[0] rescue "" - - if !_lyrics or _lyrics.to_s.empty? - _lyrics = Nerve::Services::Metadata.match_lyrics(artist.name, album.name, title)[0] \ - rescue "No lyrics available." + _lyrics = (cache_item.lyrics or JSON.parse(_big["lyrics"])[0]) rescue "" + _lyrics = _lyrics[0] if _lyrics.is_a? Array + _lyrics = '' if !_lyrics or _lyrics.to_s.empty? or _lyrics.to_s == '0' or _lyrics.to_s == 'false' + + if _lyrics == '' + _lyrics = "No lyrics available." + begin + _lyrics = Nerve::Services::Metadata.match_lyrics(artist.name, (album.name rescue ''), title)[0] + if cache_item + cache_item.lyrics = _lyrics + cache_item.save + end + rescue + _lyrics = "No lyrics available." + end end [result, _lyrics, (result["year"] rescue nil)] @@ -53,7 +71,7 @@ def is_safe r = Regexp.new("\\b((#{$config["words"]["banned"].join("|")})[^\\s\\b,.\<\>]*)\\b", 7) # If the size is 0, then it's "safe" ish - return false if !_lyrics or _lyrics.to_s.empty? + return false if !_lyrics or _lyrics.to_s.empty? or _lyrics == 0 # TODO: make this cleaner _lyrics.scan(r).size == 0 && !!_lyrics && _lyrics != "" && _lyrics != "No lyrics available." @@ -61,22 +79,25 @@ def is_safe end def set_unsafe - explicit = 1 + explicit = true end def why_unsafe - return "determined as risky during upload" if status == 0 # only works before submission - return "has parental advisory/explicit flag" if explicit + reasons = [] + reasons << "determined as risky during upload" if status == 0 # only works before submission + reasons << "has parental advisory/explicit flag" if explicit _extended, _lyrics = get_metadata - r = Regexp.new("\\b((#{$config["words"]["banned"].join("|")})[^\\s\\b,.\<\>]*)\\b", 7) - s = _lyrics.scan(r) - - return "it contains (at least one) expletive (#{s[0]})" if s.size > 0 - return "no lyrics were found" if !_lyrics or _lyrics == "No lyrics available." + if !_lyrics.to_s.empty? && _lyrics != 0 && _lyrics != "No lyrics available." + r = Regexp.new("\\b((#{$config["words"]["banned"].join("|")})[^\\s\\b,.\<\>]*)\\b", 7) + s = _lyrics.scan(r) + reasons << "it contains (at least one) expletive (#{s[0]})" if s.size > 0 + else + reasons << "no lyrics were found" + end - false + reasons.length > 0 ? reasons : false end diff --git a/db/migrate/20180217105112_update_track_cols.rb b/db/migrate/20180217105112_update_track_cols.rb new file mode 100644 index 0000000..a50e5e6 --- /dev/null +++ b/db/migrate/20180217105112_update_track_cols.rb @@ -0,0 +1,13 @@ +class UpdateTrackCols < ActiveRecord::Migration + def change + + change_column :tracks, :explicit, :boolean + change_column :tracks, :flagged, :boolean + change_column :tracks, :explicit, :boolean + change_column :tracks, :instrumental, :boolean + change_column :tracks, :is_library, :boolean + change_column :tracks, :explicit, :boolean + change_column :tracks, :is_automation, :boolean + + end +end diff --git a/db/schema.rb b/db/schema.rb index eadf54d..7387a44 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,7 +11,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180113004836) do +ActiveRecord::Schema.define(version: 20180217105112) do create_table "albums", force: :cascade do |t| t.integer "external_id", limit: 4, null: false @@ -83,14 +83,14 @@ t.integer "bitrate", limit: 4, default: 0 t.integer "sample_rate", limit: 4, default: 0 t.integer "end_type", limit: 4, default: 0 - t.integer "explicit", limit: 4, default: 0 - t.integer "is_library", limit: 4, default: 0 - t.integer "is_automation", limit: 4, default: 0 + t.boolean "explicit", default: false + t.boolean "is_library", default: false + t.boolean "is_automation", default: false t.integer "ext_id", limit: 4 t.string "playout_id", limit: 255 t.integer "restrict_play", limit: 4, default: 0 - t.integer "flagged", limit: 4, default: 0 - t.integer "instrumental", limit: 4, default: 0 + t.boolean "flagged", default: false + t.boolean "instrumental", default: false t.string "playout_id_2", limit: 255 t.text "extra", limit: 65535 t.integer "category_a", limit: 4, default: 0 diff --git a/services/metadata.rb b/services/metadata.rb index 2cf08f9..560e9e4 100644 --- a/services/metadata.rb +++ b/services/metadata.rb @@ -57,12 +57,14 @@ def self.match_lyrics artist, album, title end - def self.match_meta id, enhanced = false, column = "id" + def self.match_meta id, enhanced = false, column = "id", raw = false raise "Invalid column name specified?" if column != "id" and column != "external_id" query = Nerve::Model::CacheItem.where("#{column}=?", id) - query.count == 1 ? query.first.get_json(enhanced) : false + return false if !query.count + item = query.first.get_json(enhanced) + raw ? [item, query.first] : item end @@ -75,13 +77,7 @@ def self.get_genre id def self.cache data item = Nerve::Model::CacheItem.find_or_create_by(external_id: data['external_id']) - - data['year'] = data['year'] == '' ? 0 : data['year'] - data['track'] = data['title'] - - data = data.select {|x| item.class.attribute_names.index(x) } - item.assign_attributes data - + item.init_by data item.save item diff --git a/services/upload.rb b/services/upload.rb index 5fdb4a6..a494500 100644 --- a/services/upload.rb +++ b/services/upload.rb @@ -128,18 +128,18 @@ def self.get_total_uploads raise "You must select an end type!" \ unless [1, 2, 3].include? params['end_type'].to_i - # Reject artist name changes if params['artist'] != track.artist.name track.artist = Nerve::Model::Artist.find_or_initialize_by(name: params['artist']) track.artist.external_id ||= 0 track.artist.created_by = session[:user_id] track.set_unsafe + track.update_metadata rescue nil end - if params['title'] != track.title track.title = params['title'] track.set_unsafe + track.update_metadata rescue nil end track.end_type = params['end_type'].to_i From dfdc9c0b6fadd0cb432b8cb69e61ff2d0450f2c7 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sat, 17 Feb 2018 11:52:52 +0000 Subject: [PATCH 20/27] Bring database versions in sync --- db/migrate/20180217115144_col_ab.rb | 8 +++++ db/schema.rb | 50 ++++++++++++++++++++++++----- 2 files changed, 50 insertions(+), 8 deletions(-) create mode 100644 db/migrate/20180217115144_col_ab.rb diff --git a/db/migrate/20180217115144_col_ab.rb b/db/migrate/20180217115144_col_ab.rb new file mode 100644 index 0000000..aa9fe0e --- /dev/null +++ b/db/migrate/20180217115144_col_ab.rb @@ -0,0 +1,8 @@ +class ColAb < ActiveRecord::Migration + def change + + add_column :tracks, :category_a, :integer + add_column :tracks, :category_b, :integer + + end +end diff --git a/db/schema.rb b/db/schema.rb index 7387a44..e1c10a5 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -11,10 +11,18 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 20180217105112) do +ActiveRecord::Schema.define(version: 20180217115144) do + + create_table "album_idents", id: false, force: :cascade do |t| + t.integer "album_id", limit: 4, null: false + t.string "type", limit: 32 + t.string "identifier", limit: 255, null: false + end + + add_index "album_idents", ["identifier"], name: "in_ident", using: :btree create_table "albums", force: :cascade do |t| - t.integer "external_id", limit: 4, null: false + t.integer "external_id", limit: 4 t.string "name", limit: 255 t.integer "artist", limit: 4 t.integer "created_by", limit: 4, null: false @@ -22,7 +30,7 @@ end create_table "artists", force: :cascade do |t| - t.integer "external_id", limit: 4, null: false + t.integer "external_id", limit: 4 t.string "name", limit: 255 t.integer "created_by", limit: 4, null: false t.datetime "creation_date", null: false @@ -30,6 +38,16 @@ add_index "artists", ["name"], name: "index_artists_on_name", type: :fulltext + create_table "exports", primary_key: "export_id", force: :cascade do |t| + t.integer "track_id", limit: 4, null: false + t.datetime "export_date", null: false + t.string "export_type", limit: 32 + t.integer "playout_id", limit: 4 + end + + add_index "exports", ["playout_id"], name: "in_export", using: :btree + add_index "exports", ["track_id"], name: "in_track", using: :btree + create_table "migrate_cache", primary_key: "cart_id", force: :cascade do |t| t.string "title", limit: 255 t.string "artist", limit: 255 @@ -60,8 +78,23 @@ t.string "name", limit: 64 end + create_table "track_comments", force: :cascade do |t| + t.integer "track_id", limit: 4, null: false + t.integer "created_by", limit: 4, null: false + t.datetime "creation_date", null: false + t.text "comment", limit: 65535 + end + + create_table "track_idents", id: false, force: :cascade do |t| + t.integer "track_id", limit: 4, null: false + t.string "type", limit: 32, null: false + t.string "identifier", limit: 255, null: false + end + + add_index "track_idents", ["identifier"], name: "in_ident", using: :btree + create_table "tracks", force: :cascade do |t| - t.integer "external_id", limit: 4, null: false + t.integer "external_id", limit: 4 t.string "title", limit: 255 t.integer "artist", limit: 4, null: false t.integer "album", limit: 4, null: false @@ -73,7 +106,7 @@ t.datetime "last_update" t.datetime "local_kill_date" t.string "local_path", limit: 255 - t.decimal "intro_start", precision: 6, scale: 2, default: 0.0 + t.decimal "intro_start", precision: 6, scale: 2 t.decimal "intro_end", precision: 6, scale: 2 t.decimal "hook_end", precision: 6, scale: 2 t.decimal "hook_start", precision: 6, scale: 2 @@ -86,15 +119,16 @@ t.boolean "explicit", default: false t.boolean "is_library", default: false t.boolean "is_automation", default: false - t.integer "ext_id", limit: 4 + t.string "ext_id", limit: 255 t.string "playout_id", limit: 255 t.integer "restrict_play", limit: 4, default: 0 + t.string "note", limit: 255 t.boolean "flagged", default: false t.boolean "instrumental", default: false t.string "playout_id_2", limit: 255 t.text "extra", limit: 65535 - t.integer "category_a", limit: 4, default: 0 - t.integer "category_b", limit: 4, default: 0 + t.integer "category_a", limit: 4 + t.integer "category_b", limit: 4 end add_index "tracks", ["title"], name: "index_tracks_on_title", type: :fulltext From 6b7d55a80545e2cd3af17e14d17f409249681a2e Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 18 Feb 2018 21:48:53 +0000 Subject: [PATCH 21/27] Hotfix: fix auto approval not working if there is no user with ID 0 in the database --- app/models/user.rb | 2 +- services/upload.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/user.rb b/app/models/user.rb index 78d0f32..e9f8fc1 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -27,7 +27,7 @@ def after_initialize 'safety_net' => true, } - permissions.each { | k, v| @_permissions[k] = v } + permissions.each { | k, v| @_permissions[k] = v } if permissions.is_a? Hash if admin or moderator @_permissions['override_bitrate'] = true diff --git a/services/upload.rb b/services/upload.rb index a494500..b0a7225 100644 --- a/services/upload.rb +++ b/services/upload.rb @@ -193,7 +193,7 @@ def self.get_total_uploads if track.is_safe track.status = 4 - track.approved_by = 0 + track.approved_by = User.find_or_create_by(id: 0) track.save Nerve::Job::Transfer.create({ "track_id" => id}) From 8ed94c2b58ddfd0aaeabecc8da217db5443d2954 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 18 Feb 2018 21:49:45 +0000 Subject: [PATCH 22/27] As before, but actually use the correct namespace --- services/upload.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/upload.rb b/services/upload.rb index b0a7225..4c1a881 100644 --- a/services/upload.rb +++ b/services/upload.rb @@ -193,7 +193,7 @@ def self.get_total_uploads if track.is_safe track.status = 4 - track.approved_by = User.find_or_create_by(id: 0) + track.approved_by = Nerve::Model::User.find_or_create_by(id: 0) track.save Nerve::Job::Transfer.create({ "track_id" => id}) From 84012caaa2b2dd8384b7f4a5f830181cfb9415d6 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Sun, 18 Feb 2018 22:25:44 +0000 Subject: [PATCH 23/27] Quick fix invalid comparison --- services/metadata.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/metadata.rb b/services/metadata.rb index 560e9e4..e60a32f 100644 --- a/services/metadata.rb +++ b/services/metadata.rb @@ -62,7 +62,7 @@ def self.match_meta id, enhanced = false, column = "id", raw = false raise "Invalid column name specified?" if column != "id" and column != "external_id" query = Nerve::Model::CacheItem.where("#{column}=?", id) - return false if !query.count + return false if query.count == 0 or !query.first item = query.first.get_json(enhanced) raw ? [item, query.first] : item @@ -133,4 +133,4 @@ def self.cache data end end -end \ No newline at end of file +end From c4b2264e6f42c0249125e0f4689f60a2793a093a Mon Sep 17 00:00:00 2001 From: Jamie Woods Date: Mon, 19 Mar 2018 11:13:51 +0000 Subject: [PATCH 24/27] Argh! --- jobs/cleanup.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jobs/cleanup.rb b/jobs/cleanup.rb index 1f5a9f3..4c7fd56 100644 --- a/jobs/cleanup.rb +++ b/jobs/cleanup.rb @@ -23,7 +23,7 @@ def perform @original_status = track.status - track.delete! track.status == 5 + track.delete! track.status < 4 i += 1 at(i * multiplier, 100) From b1c38cfabe6565f870235879d189c7443919d615 Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Wed, 28 Mar 2018 12:09:52 +0100 Subject: [PATCH 25/27] Rebuild preview cheat fix --- app.rb | 9 ++--- jobs/process.rb | 6 ++-- scripts/rebuild_preview.rb | 67 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 6 deletions(-) create mode 100644 scripts/rebuild_preview.rb diff --git a/app.rb b/app.rb index d139f84..35ddf3c 100644 --- a/app.rb +++ b/app.rb @@ -9,14 +9,15 @@ require 'sinatra/xsendfile' require 'yaml' -require './modules' +$: << '.' +require_relative 'modules' Bundler.require -$config = YAML::load(File.read("config.yml")) -$genres = YAML::load(File.read("genres.yml")) +$config = YAML::load(File.read(File.join(File.dirname(__FILE__), "config.yml"))) +$genres = YAML::load(File.read(File.join(File.dirname(__FILE__), "genres.yml"))) $env = {:slave => false} -require './database' +require_relative 'database' Resque::Plugins::Status::Hash.expire_in = (60 * 60) * 24 diff --git a/jobs/process.rb b/jobs/process.rb index ae38522..5c435e2 100644 --- a/jobs/process.rb +++ b/jobs/process.rb @@ -245,12 +245,14 @@ def tag file, data - def generate_waveform file, length + def generate_waveform file, length, output = nil + + output = "#{file}.dat" if output == nil _debug "Generating a waveform of the input - hang on" pps = (20000 / length).to_i result = _sys('audiowaveform', - '-i', file, '-o', @wave_path = "#{file}.dat", + '-i', file, '-o', @wave_path = output, '-b', '8', '--pixels-per-second', pps.to_s) diff --git a/scripts/rebuild_preview.rb b/scripts/rebuild_preview.rb new file mode 100644 index 0000000..9c13a1b --- /dev/null +++ b/scripts/rebuild_preview.rb @@ -0,0 +1,67 @@ +require 'json' +require 'rest-client' +require 'base64' + +$: << '..' +require_relative '../app' + +#$config = YAML::load(File.read("../config.yml")) + +class Processor < Nerve::Job::Process + + include Object.const_get($config["export"]["mode"]) + + def initialize; end + + def process_track track + + # rebuild the track's preview and peaks + path = track.local_path + prev_path = $config["export"]["directory"] + "/" + path + ".ogg" + wave_path = $config["export"]["directory"] + "/" + path + ".dat" + + # If we already have the preview, we probably + if File.exist?(prev_path) or !File.exist?(track) + _debug "Skipping for some reason" + return + end + + exported_path = self.find_path(track) + + _debug "Generating preview" + + convert(exported_path, prev_path, "ogg", "-1") + + raise "Conversion failed!" unless File.exists? prev_path + + _debug "Generating waveform" + + generate_waveform(prev_path, options["length"], wave_path) + + raise "Generating waveform failed" unless File.exists? + + _debug "Done!" + + end + +end + + +processor = Processor.new + +tracks = Nerve::Model::Track.all +tracks.each { | t | + + begin + processor.process_track t + rescue + begin + processor.process_track t + rescue + p "Track #{t.id} failed" + p $! + p $!.backtrace + end + end + sleep 0.2 +} \ No newline at end of file From 477691a13d4d419bc6a0d107462622ad6f17034d Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Wed, 28 Mar 2018 12:25:18 +0100 Subject: [PATCH 26/27] Make work --- scripts/rebuild_preview.rb | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/scripts/rebuild_preview.rb b/scripts/rebuild_preview.rb index 9c13a1b..fddb1a5 100644 --- a/scripts/rebuild_preview.rb +++ b/scripts/rebuild_preview.rb @@ -13,6 +13,8 @@ class Processor < Nerve::Job::Process def initialize; end + def at *args; end + def process_track track # rebuild the track's preview and peaks @@ -20,27 +22,45 @@ def process_track track prev_path = $config["export"]["directory"] + "/" + path + ".ogg" wave_path = $config["export"]["directory"] + "/" + path + ".dat" + exported_path = Object.const_get($config["export"]["mode"]).find_path(track) + # If we already have the preview, we probably - if File.exist?(prev_path) or !File.exist?(track) + if File.exist?(prev_path) or !File.exist?(exported_path) _debug "Skipping for some reason" return end - exported_path = self.find_path(track) - _debug "Generating preview" + temp = [] + + begin + + _debug "Generating preview" + + temp << Tempfile.new(['nerve', '.wav']) + temp_path = temp[0].path + temp[0].close; temp[0].unlink + + FileUtils::copy(exported_path, temp_path) + + convert(temp_path, prev_path, "ogg", "-1") - convert(exported_path, prev_path, "ogg", "-1") + raise "Conversion failed!" unless File.exists? prev_path - raise "Conversion failed!" unless File.exists? prev_path + _debug "Generating waveform" - _debug "Generating waveform" + generate_waveform(temp_path, track.length.to_f, wave_path) - generate_waveform(prev_path, options["length"], wave_path) + raise "Generating waveform failed" unless File.exists? wave_path - raise "Generating waveform failed" unless File.exists? + _debug "Done!" + + rescue - _debug "Done!" + temp.map { |t| File.unlink(t) } + raise $! + + end end From 4d664734ef33c6bf093bd254581fe17f12e9196b Mon Sep 17 00:00:00 2001 From: Jamie0 Date: Wed, 28 Mar 2018 12:28:03 +0100 Subject: [PATCH 27/27] Ensure the temp files get cleaned --- scripts/rebuild_preview.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/rebuild_preview.rb b/scripts/rebuild_preview.rb index fddb1a5..b1c2553 100644 --- a/scripts/rebuild_preview.rb +++ b/scripts/rebuild_preview.rb @@ -55,10 +55,9 @@ def process_track track _debug "Done!" - rescue + ensure temp.map { |t| File.unlink(t) } - raise $! end