Merge pull request #3 from lcdhoffman/master

1.0.0-rc1

README.md

@@ -1,19 +1,31 @@
 Archon2ArchivesSpace README
 ================
+# System Requirements
+
+You will need to have Ruby 1.9.3 installed to run this service
+
+	ruby --version
+    # example output: ruby 1.9.3p429 (2013-05-15 revision 40747)
+
+If your system has a different version of Ruby installed, the simplest way to
+leave your system intact and get 1.9.3 is to install RVM (https://rvm.io/).
+
 # Installing the service
 
-Checkout the project from Github:
+Download a release or just checkout the project from Github:
 
     git clone https://github.com/lcdhoffman/archon-migration.git
     cd archon-migration
 
 Run a script to download the necessary ArchivesSpace libraries:
 
-./scripts/import\_client\_libs.sh v1.0.0RC1
+	./scripts/import\_client\_libs.sh v1.0.0RC1
 
 This will attempt to download the ArchivesSpace source code for ArchivesSpace v1.0.0RC1.
+*Note: the service ships with libraries for ArchivesSpace 1.0.0, so you can skip this step
+if you are targeting 1.0.0.
 
-Install the gems in the Gemfile:
+Install the application dependencies listed in the Gemfile:
 
     gem install bundler
     bundle install
@@ -22,7 +34,17 @@ Now run the application:
 
     ruby app/main.rb
 
-(You'll probably want to daemonize or disown this.)
+The service runs on port 4568 by default. To change this:
+
+	touch config/config_local.rb
+    echo "Appdata.port_number YOUR_FAVORITE_PORT_HERE" >> config/config_local.rb
+
+# Daemonizing the Service
+
+The service can be daemonized in several ways. One option is to install a native
+ruby solution such as the Daemonize gem (http://daemons.rubyforge.org/). However,
+since the service is intended to be short-lived, it may be easiest to simply
+send the process to the background and disown it.
 
 # Using the Service
 
@@ -40,10 +62,10 @@ The best way to configure the application is to create a local config file:
 
     touch config/config_local.rb
 
-To change, for example, the port that application runs on, add the following
+To change, for example, the version of the ArchivesSpace target, add the following
 line
 
-    Appdata.port_number 4568
+	Appdata.aspace_version 'v1.0.1'
 
 # Notes
 
@@ -60,10 +82,12 @@ contains data.
 
 Do not allow Archon users to create or edit data while the migration is running.
 
+Do not allow ArchivesSpace users to create or edit data while the migration is
+running.
+
 You can optimize the performance of the migration tool by adjusting the number of
 pages of Archon data that are cached. For example, if your largest Archon collection contains 50,000 Content records, and you are running the migration tool in an environment that can afford around 300MB of memory, you might want to add this line to your config_local.rb file:
 
     Appdata.archon_page_cache_size 500
 
-There's no (or little) advantage to setting the cache size to a value larger than the number of Content records in the largest Collection, divided by 100.
-
+There's no (or little) advantage to setting the page cache size to a value larger than the number of Content records in the largest Collection, divided by 100. There is a significant disadvantage to keeping your page cache size smaller than the number of pages of items in your largest collection.

app/js/main.js

@@ -4,13 +4,15 @@ $(document).ready(function(){
     beforeSubmit: function(arr, $form, options){
       $form.validate();
       $('#start-migration').toggleClass('pure-button-disabled');
+      $('#download-files').addClass('pure-button-disabled');
+      $('#download-log').addClass('pure-button-disabled');
       $('#status-console').empty();
 //      $('#status-console').append("<progress></progress>");
     },
     success:      function(responseText, statusText, xhr, $form){
       $('#start-migration').toggleClass('pure-button-disabled');
-      $('#download-files').toggleClass('pure-button-disabled');
-      $('#download-log').toggleClass('pure-button-disabled');
+      $('#download-files').removeClass('pure-button-disabled');
+      $('#download-log').removeClass('pure-button-disabled');
     },
     xhr:          function(){
       var xhr = new XMLHttpRequest();
@@ -33,21 +35,18 @@ $(document).ready(function(){
     // If checked
     if ($("#nodourl").is(":checked")) {
     //show the hidden div
-      console.log("remove");
       $("#do_baseurl").removeAttr('required');
-      console.log($("#do_baseurl").attr('required'));
     }  else {
       //otherwise, hide it
       $("#do_baseurl").attr("required", "true");
-      console.log($("#do_baseurl").attr('required'));
     }
   });
 
 });
 
 
 function updateStatus(update, emitter){
-    console.log(update);
+//    console.log(update);
     if (update.type == 'error') {
       emitter.show_error(update.body);
     } else if (update.type == 'status') {
@@ -58,6 +57,8 @@ function updateStatus(update, emitter){
       emitter.show_progress(update.ticks, update.total);
     } else if (update.type == 'update') {
       emitter.show_progress_message(update.body);
+    } else if (update.type == 'log') {
+      $('#download-log').attr('href', update.file);
     } else {
       // todo: toggle in progress bar
     }
@@ -68,11 +69,12 @@ function StatusEmitter() {
   var console = $('#status-console');
 
   this.refresh_status = function(status, source){
-    $("#status-console p:last .progress").html(' 100%');
-    if (source == 'aspace' && $("#status-console p:last").hasClass('aspace')) {
-      $("#status-console p:last").html(status);
+    if (source == 'aspace') {
+      $("#status-console div:last p.aspace").html(status);
     } else {
-      console.append("<p class=\"status " + source + "\">"+status+"</p>");
+      $("#status-console div:last p.aspace").remove();
+      $("#status-console div:last span.progress-message").html(" - Done");
+      console.append("<div class=\"status " + source + "\"><p class=\"main\">"+status+"</p><p class=\"aspace\"></p></div>");
     }
   }
 
@@ -87,13 +89,13 @@ function StatusEmitter() {
 
   this.show_progress = function(ticks, total) {
     var percent = Math.round((ticks / total) * 100);
-    $("#status-console p:last .progress").remove();
-    $("#status-console p:last").append("<span class='progress'> " + percent + "%</span>");
+    $("#status-console div:last span.progress").remove();
+    $("#status-console div:last p:last").append("<span class='progress'> " + percent + "%</span>");
   }
 
   this.show_progress_message = function(body) {
-    $("#status-console p:last .progress-message").remove();
-    $("#status-console p:last").append("<span class='progress-message'> - " + body + "</span>");
+    $("#status-console div:last p:first span.progress-message").remove();
+    $("#status-console div:last p:first").append("<span class='progress-message'> - " + body + "</span>");
   }
 }
 

app/lib/archivesspace_client.rb

@@ -45,7 +45,9 @@ def init_session
       req.form_data = {:password => @password}
 
       response = JSONModel::HTTP.do_http_request(url, req)
-      raise "Session Init error" unless response.code == '200'
+      unless response.code == '200'
+        raise "Couldn't log into ArchivesSpace and start a session"
+      end
 
       json = JSON::parse(response.body)
       @session = json['session']
@@ -102,6 +104,16 @@ def repo(id)
       self
     end
 
+    # rely on client tools to break when uris don't exist
+    def database_empty? 
+      begin 
+        response = get_json('/repositories/2')
+        return false
+      rescue NoMethodError
+        return true
+      end
+    end
+
 
     def import(y)
 

app/lib/archon_client.rb

@@ -20,6 +20,7 @@ def self.record_type(key, &block)
 
 
   def self.init
+    @@record_types = {}
     Dir.glob(File.join(File.dirname(__FILE__),
                        '../',
                        'models', 

app/lib/migrate.rb

@@ -56,16 +56,19 @@ def initialize(params = {})
   end
 
 
-  def connection_check
-    if @archon.has_session? && @aspace.has_session?
-      $log.debug("All systems go")
-    else
-      $log.warn("Not connected")
+  def all_good_or_die
+    if !@archon.has_session? 
+      raise "No Archon connection"
+    elsif !@aspace.has_session?
+      raise "No ArchivesSpace connection"
+    elsif !@aspace.database_empty?
+      raise "The ArchivesSpace instance you are connecting to is not empty. Please backup and delete the database"
     end
   end
 
 
   def migrate(y)
+    all_good_or_die
     @y = y
     Thread.current[:selected_repo_id] = 1
 
@@ -86,6 +89,7 @@ def migrate(y)
 
     # 5: Package Digital File content for Download
     package_digital_files
+    emit_status("Migration complete. Download the log to review warnings")
   end
 
 
@@ -174,21 +178,29 @@ def migrate_collection_content(repo_id,
 
     @aspace.repo(repo_id).import(@y) do |batch|
       container_trees = {}
+      position_tracker = {}
+      position_map = {}
 
       Archon.record_type(:content).set(coll_id).each do |rec|
         import_id = rec.class.import_id_for(rec['ID'])
         rec.class.transform(rec) do |obj_or_cont|
           if obj_or_cont.is_a?(Array)
-            unless container_trees.has_key?(obj_or_cont[0])
-              container_trees[obj_or_cont[0]] = []
+            cont = obj_or_cont
+            unless container_trees.has_key?(cont[0])
+              container_trees[cont[0]] = []
             end
-            container_trees[obj_or_cont[0]] << obj_or_cont[1]
+            container_trees[cont[0]] << cont[1]
           else
+            obj = obj_or_cont
+            set_key = obj.parent.nil? ? nil : obj.parent['ref']
+            position_tracker[set_key] ||= {}
+            position_tracker[set_key][obj.position] = obj.key
+
             resolve_ids_to_links(rec, obj_or_cont, classification_map)
 
             # link resource
             resource_uri = resource_map[import_id_for(:collection, coll_id)]
-            obj_or_cont.resource = {:ref => resource_uri}
+            obj.resource = {:ref => resource_uri}
             # attach digital object instances
             if digital_instance_map && digital_instance_map[import_id]
               digital_instance_map[import_id].each do |do_uri|
@@ -197,15 +209,29 @@ def migrate_collection_content(repo_id,
                 instance.digital_object = {
                   :ref => do_uri
                 }
-                obj_or_cont.instances << instance
+                obj.instances << instance
               end
             end
 
             batch.unshift(obj_or_cont)
           end
         end
       end
+
+      # it might not be a bad idea to move this
+      # to aspace one day
+      position_tracker.each do |id, map|
+        sorted = map.keys.sort
+        sorted.each_with_index do |padded_position, real_position|
+          position_map[map[padded_position]] = real_position
+        end
+      end
+
       batch.each do |obj|
+        if position_map.has_key?(obj.key)
+          obj.position = position_map[obj.key]
+        end
+
         if (container_data_sets = container_trees[obj.key])
           container_data_sets.each do |container_data|
             container = ASpaceImport.JSONModel(:container).new

app/main.rb

@@ -52,8 +52,14 @@
   Enumerator.new do |y|
     begin
       stamp = Time.now.strftime("%Y-%m-%d-%H-%M-%S")
-      $syslog = Logger.new(Appdata.app_dir + "/public/log-#{stamp}.txt")
+      $logfile = File.new(Appdata.app_dir + "/public/log-#{stamp}.txt", 'w')
+      $syslog = Logger.new($logfile)
       $log = MigrationLog.new(y, $syslog)
+      y << JSON.generate({
+                           :type => :log, 
+                           :file => File.basename($logfile.path)
+                         }) + "---\n"
+
       m = MigrationJob.new(params[:job])
       m.migrate(y)
     rescue JSONModel::ValidationException => e
@@ -68,6 +74,7 @@
       y << JSON.generate({:type => :error, :body => e.to_s}) + "---\n"
     ensure
       $log = $syslog
+      $log.close
     end
   end
 end

app/models/archon_collection.rb

@@ -87,15 +87,6 @@ def self.transform(rec)
     obj.finding_aid_author = rec['FindingAidAuthor']
 
     #Notes
-    if rec['Scope']
-      obj.notes << model(:note_singlepart,
-                         {
-                           :content => [rec['Scope']],
-                           :label => 'Scope and Contents',
-                           :type => unspecified('abstract')
-                         })
-    end
-
     if rec['Abstract']
       obj.notes << model(:note_singlepart,
                          {
@@ -285,6 +276,16 @@ def self.note_mappings
      {:archon_type => %w(RelatedMaterials RelatedMaterialsURL),
        :note_type => 'relatedmaterial',
        :label => "Related Materials"
+     },
+
+     {:archon_type => 'Scope',
+       :note_type => 'scopecontent',
+       :label => "Scope and Contents"
+     },
+
+     {:archon_type => 'ProcessingInfo',
+       :note_type => 'processinfo',
+       :label => "Processing Information"
      }
 
     ]