Bots and data export

Gemfile

@@ -38,6 +38,10 @@ gem 'puma', '~> 2.14.0'
 
 gem 'logstasher', '~> 0.6'
 
+gem 'aws-sdk', '~> 2'
+
+# gem 'mongoid_fulltext'
+
 group :development do
   gem 'dotenv-rails'
 end

Gemfile.lock

@@ -52,6 +52,12 @@ GEM
       tzinfo (~> 0.3.37)
     addressable (2.3.8)
     arel (4.0.2)
+    aws-sdk (2.2.14)
+      aws-sdk-resources (= 2.2.14)
+    aws-sdk-core (2.2.14)
+      jmespath (~> 1.0)
+    aws-sdk-resources (2.2.14)
+      aws-sdk-core (= 2.2.14)
     bcrypt (3.1.10)
     better_errors (2.1.1)
       coderay (>= 1.0.0)
@@ -126,6 +132,7 @@ GEM
     jbuilder (1.5.3)
       activesupport (>= 3.0.0)
       multi_json (>= 1.2.0)
+    jmespath (1.1.3)
     jquery-rails (3.1.2)
       railties (>= 3.0, < 5.0)
       thor (>= 0.14, < 2.0)
@@ -303,6 +310,7 @@ PLATFORMS
 DEPENDENCIES
   actionpack-action_caching
   active_model_serializers
+  aws-sdk (~> 2)
   better_errors
   binding_of_caller
   browserify-rails (~> 0.9.1)

app/assets/javascripts/components/app-router.cjsx

@@ -11,6 +11,10 @@ Verify                        = require './verify'
 # TODO Group routes currently not implemented
 GroupPage                     = require './group-page'
 GroupBrowser                  = require './group-browser'
+FinalSubjectSetBrowser        = require './final-subject-set-browser'
+FinalSubjectSetPage           = require './final-subject-set-page'
+FinalSubjectSetDownload       = require './final-subject-set-download'
+GenericPage                   = require './generic-page'
 
 Project                       = require 'models/project.coffee'
 
@@ -74,16 +78,36 @@ class AppRouter
               name={workflow.name + '_entire_page'}
             />
         }
+
+
         { # Project-configured pages:
           project.pages?.map (page, key) =>
             <Route
               key={key}
-              path={page.name}
+              path={page.key}
               handler={@controllerForPage(page)}
               name={page.name}
             />
         }
 
+        <Route
+          path="#{project.data_url_base}/browse"
+          handler={FinalSubjectSetBrowser}
+          name='final_subject_sets'
+        />
+        { if project.downloadable_data
+          <Route
+            path="#{project.data_url_base}/browse/:final_subject_set_id"
+            handler={FinalSubjectSetPage}
+            name='final_subject_set_page'
+          />
+        }
+        <Route
+          path="#{project.data_url_base}/download"
+          handler={FinalSubjectSetDownload}
+          name='final_subject_sets_download'
+        />
+
         <Route
           path='groups'
           handler={GroupBrowser}
@@ -115,7 +139,6 @@ class AppRouter
         #   $("div#" + selectedID).addClass("selected-content"))
         #   $("a#" + selectedID).addClass("selected-content"))
 
-
       componentDidMount: ->
         pattern = new RegExp('#/[A-z]*#(.*)')
         selectedID = "#{window.location.hash}".match(pattern)
@@ -141,21 +164,11 @@ class AppRouter
           active: false
           heightStyle: "content"
 
-      navToggle:(e)->
-
       render: ->
         formatted_name = page.name.replace("_", " ")
-        <div className="page-content custom-page" id="#{page.name}">
-          <h1>{formatted_name}</h1>
-          <div dangerouslySetInnerHTML={{__html: marked(page.content)}} />
-          {
-            if page.group_browser? && page.group_browser != ''
-              <div className='group-area'>
-                <GroupBrowser project={project} title={page.group_browser} />
-              </div>
-          }
-          <div className="updated-at">Last Update {page.updated_at}</div>
-        </div>
+        base_key = page.key.split('/')[0]
+        nav = project.page_navs[base_key]
+        <GenericPage key={page.name} title={formatted_name} nav={nav} current_nav={"/#/#{page.key}"} content={page.content} footer={"Last Update: #{moment(page.updated_at, moment.ISO_8601).calendar()}"} />
 
 module.exports = AppRouter
 window.React = React

app/assets/javascripts/components/final-subject-assertion.cjsx

@@ -0,0 +1,75 @@
+React = require 'react'
+API   = require '../lib/api'
+
+module.exports = React.createClass
+  displayName: 'FinalSubjectAssertion'
+
+  propTypes: ->
+    assertion:    React.PropTypes.object.isRequired
+
+  getInitialState: ->
+    showingRegion: false
+
+  toggleRegion: (e) ->
+    console.log "show: ", ! @state.showingRegion
+    @setState showingRegion: ! @state.showingRegion
+
+  render: ->
+
+    confidence = Math.round(100 * @props.assertion.confidence)
+    confidence_label = 'low'
+    confidence_label = 'med' if confidence >= 50
+    confidence_label = 'high' if confidence >= 66
+    confidence_label = 'max' if confidence == 100
+
+    status_label = @props.assertion.status.replace /_/, ' '
+
+    <div className="confidence-#{confidence_label} status-#{@props.assertion.status}">
+      <h3>{@props.assertion.name}</h3>
+
+      <ul className="assertion-data">
+      { for k of @props.assertion.data
+          <li key={k}>
+            {
+              cleaned_version = null
+              if @props.field && @props.field.value
+                cleaned_version = if (typeof @props.field.value) == 'object' then @props.field.value[k] else @props.field.value
+              null
+            }
+            <span className="value">{@props.assertion.data[k]}</span>
+            { if cleaned_version && ('' + cleaned_version) != @props.assertion.data[k]
+                <span className="cleaned-version">( Interpretted as <em>{if (typeof cleaned_version) == 'object' then cleaned_version.join(' x ') else cleaned_version }</em> )</span>
+            }
+            { if k != 'value'
+                <span className="data-key">({k.replace /_/g, ' '})</span>
+            }
+          </li>
+      }
+      </ul>
+      <dl className="assertion-properties">
+        <dt className="confidence">Confidence</dt>
+        <dd className="confidence">{confidence}%</dd>
+        <dt className="status">Status</dt>
+        <dd className="status">{status_label}</dd>
+        <dt>Distinct Transcriptions</dt>
+        <dd>{@props.assertion.versions?.length || 0}</dd>
+      </dl>
+      <a className="show-region-link" href="javascript:void(0);" onClick={@toggleRegion}>
+      { if @state.showingRegion
+          <span>Hide {@props.project.term('mark')}</span>
+        else
+          <span>Show {@props.project.term('mark')}</span>
+      }
+      </a>
+      {
+        viewer_width = @props.assertion.region.width
+        scale = viewer_width / @props.assertion.region.width
+        s =
+          background: "url(#{@props.subject.location.standard}) no-repeat -#{Math.round(@props.assertion.region.x * scale)}px -#{Math.round(@props.assertion.region.y * scale)}px"
+          width: viewer_width + 'px'
+          height: (if @state.showingRegion then Math.round(@props.assertion.region.height * scale) else 0) + 'px'
+        classes = ['image-crop']
+        classes.push 'showing' if @state.showingRegion
+        <div className={classes.join ' '} image-crop" src={@props.subject.location.standard} style={s} />
+      }
+    </div>

app/assets/javascripts/components/final-subject-set-browser.cjsx

@@ -0,0 +1,194 @@
+React                     = require 'react'
+{Navigation}              = require 'react-router'
+API                       = require '../lib/api'
+Project                   = require 'models/project.coffee'
+GenericButton             = require('components/buttons/generic-button')
+LoadingIndicator          = require('components/loading-indicator')
+Pagination                = require('components/pagination')
+GenericPage               = require './generic-page'
+FetchProjectMixin         = require 'lib/fetch-project-mixin'
+
+module.exports = React.createClass
+  displayName: 'FinalSubjectSetBrowser'
+
+  mixins: [Navigation, FetchProjectMixin]
+
+  getInitialState:->
+    entered_keyword: @props.query.keyword
+    selected_field: @props.query.field
+    searched_query: {}
+    fetching_keyword: null
+    current_page: @props.query.page ? 1
+    more_pages: false
+    results: []
+    project: null
+
+  componentDidMount: ->
+    @checkQueryString()
+
+  componentWillReceiveProps: (new_props) ->
+    @checkQueryString new_props
+
+  checkQueryString: (props = @props) ->
+    if props.query.keyword
+      @fetch({keyword: props.query.keyword, field: props.query.field}, props.query.page)
+
+  fetch: (query, page = 1) ->
+    return if ! @isMounted()
+
+    if query.keyword != @state.searched_keyword || query.field != @state.selected_field || @props.current_page != page
+
+      results = @state.results
+      results = [] if @state.searched_query?.keyword != query.keyword
+      @setState fetching_keyword: query.keyword, fetching_page: page, results: results, () =>
+        per_page = 20
+        params =
+          keyword: query.keyword
+          field: query.field
+          per_page: per_page
+          page: @state.fetching_page
+
+        API.type('final_subject_sets').get(params).then (sets) =>
+          @setState
+            results: sets
+            searched_query:
+              keyword: @props.query.keyword
+              field: @props.query.field
+            current_page: page
+            fetching_page: null
+            more_pages: sets?[0]?.getMeta('next_page')
+            fetching_keyword: null
+
+  handleKeyPress: (e) ->
+    if @isMounted()
+
+      if [13].indexOf(e.keyCode) >= 0 # ENTER:
+        @search e.target.value
+
+  search: (keyword, search_field) ->
+    keyword = @state.entered_keyword # refs.search_input?.getDOMNode().value.trim() unless keyword?
+    field = @state.selected_field # @refs.search_field?.getDOMNode().value.trim()
+
+    @transitionTo "final_subject_sets", null, {keyword: keyword, field: field}
+
+  loadMore: ->
+    @fetch @state.searched_query, @state.current_page + 1
+
+  handleChange: (e) ->
+    @setState entered_keyword: e.target.value
+
+  handleFieldSelect: (e) ->
+    @setState selected_field: e.target.value
+
+
+  renderPagination: ->
+    <Pagination
+      total_pages       = {@state.results[0]?.getMeta('total_pages')}
+      current_page      = {@state.results[0]?.getMeta('current_page')}
+      next_page         = {@state.results[0]?.getMeta('next_page')}
+      prev_page         = {@state.results[0]?.getMeta('prev_page')}
+      onClick           = {@goToPage}
+    />
+
+  renderSearch: ->
+    <div>
+      <p>Preview the data by searching by keyword below:</p>
+      <form>
+        { if @state.project.export_document_specs?[0]?.spec_fields
+            <select ref="search_field" value={@state.selected_field} onChange={@handleFieldSelect}>
+              <option value="">All Fields</option>
+              { for field in @state.project.export_document_specs[0].spec_fields when typeof(field.format)== 'string'
+                  <option key={field.name} value={field.name}>{field.name}</option>
+              }
+            </select>
+        }
+        <div>
+          <input id="data-search" type="text" placeholder="Enter keyword" ref="search_input" value={@state.entered_keyword} onChange={@handleChange} onKeyDown={@handleKeyPress} />
+          <button className="standard-button" onClick={@search}>Search</button>
+        </div>
+      </form>
+
+      { if @state.fetching_keyword
+          <LoadingIndicator />
+
+        else if @state.searched_query?.keyword && @state.results.length == 0
+          <p>No matches yet for "{@state.searched_query.keyword}"</p>
+
+        else if @state.results.length > 0
+          <div>
+            <p>Found {@state.results[0].getMeta('total')} matches</p>
+
+            <ul className="results">
+            { for set in @state.results
+                url = "/#/#{@state.project.data_url_base}/browse/#{set.id}?keyword=#{@state.searched_query.keyword}&field=#{@state.searched_query.field ? ''}"
+                matches = []
+
+                safe_keyword = (w.replace(/\W/g, "\\$&") for w in @state.searched_query.keyword.toLowerCase().replace(/"/g,'').split(' ')).join("|")
+                safe_keyword = (c for c in safe_keyword).join ",?"
+                regex = new RegExp("(#{safe_keyword})", 'gi')
+
+                # If a specific field searched, always show that:
+                if @state.searched_query?.field
+                  term = set.search_terms_by_field[@state.searched_query.field]?.join("; ")
+                  matches.push(field: @state.searched_query.field, term: term) if term
+
+                # Otherwise show all fields that match
+                else
+                  for k of set.search_terms_by_field
+                    matches.push(field: k, term: v) for v in set.search_terms_by_field[k] when v.match(regex)
+
+                <li key={set.id}>
+                  <div className="image">
+                    <a href={url}>
+                      <img src={set.subjects[0]?.location.thumbnail} />
+                    </a>
+                  </div>
+                  <div className="matches">
+                    { for m,i in matches[0...2]
+                        <div key={i} className="match">
+                          <a href={url}>
+                            <span className="field">{m.field}</span>
+                            <span className="term" dangerouslySetInnerHTML={{__html: m.term.truncate(100).replace(regex, "<em>$1</em>")}} />
+                          </a>
+                        </div>
+                    }
+                  </div>
+                </li>
+            }
+            </ul>
+
+            { @renderPagination() if @state.results.length > 0 }
+          </div>
+      }
+    </div>
+
+
+  render: ->
+    return null if ! @state.project?
+
+    data_nav = @state.project.page_navs[@state.project.data_url_base]
+
+    <GenericPage key='final-subject-set-browser' title="Data Exports" nav={data_nav} current_nav="/#/#{@state.project.data_url_base}/browse">
+      <div className="final-subject-set-browser">
+
+        <h2>Browse</h2>
+
+        { if ! @state.project.downloadable_data
+            <div>
+              <h3>Data Exports Not Available</h3>
+              <p>Sorry, but public data exports are not enabled for this project yet.</p>
+            </div>
+
+          else
+            <div>
+              { if ! @state.searched_query?.keyword
+                  <p>Participants have made {@state.project.classification_count.toLocaleString()} contributions to {@state.project.title} to date. This project periodically builds a merged, anonymized dump of that data, which is made public here.</p>
+              }
+
+              { @renderSearch() }
+
+            </div>
+        }
+      </div>
+    </GenericPage>
+