From 49aee019c7ff8a96122f39e96e24e6da6900a471 Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Mon, 15 Oct 2018 15:40:26 -0700
Subject: [PATCH 1/6] allow for ignoring temperature data when considering
 device report equivalence

(part of #461, #458)
---
 json-schema/input.yaml                    |  2 +
 lib/Conch/Controller/DeviceReport.pm      |  1 +
 lib/Conch/DB/Result/Device.pm             | 16 +++++--
 lib/Conch/DB/ResultSet/DeviceReport.pm    | 58 +++++++++++++++++++++++
 t/integration/04_test_datacenter_loaded.t | 27 ++++++++++-
 5 files changed, 98 insertions(+), 6 deletions(-)
 create mode 100644 lib/Conch/DB/ResultSet/DeviceReport.pm

diff --git a/json-schema/input.yaml b/json-schema/input.yaml
index 7b4f80931..62436cb54 100644
--- a/json-schema/input.yaml
+++ b/json-schema/input.yaml
@@ -121,6 +121,7 @@ definitions:
               drive_type:
                 type: string
               temp:
+                # TODO: move to different endpoint
                 $ref: /definitions/int_or_stringy_int
               enclosure:
                 type: string
@@ -219,6 +220,7 @@ definitions:
       system_uuid:
         $ref: /definitions/uuid
       temp:
+        # TODO: move to different endpoint
         type: object
         required:
           - cpu0
diff --git a/lib/Conch/Controller/DeviceReport.pm b/lib/Conch/Controller/DeviceReport.pm
index 9779261f9..54095d19c 100644
--- a/lib/Conch/Controller/DeviceReport.pm
+++ b/lib/Conch/Controller/DeviceReport.pm
@@ -89,6 +89,7 @@ sub process ($c) {
 	my $existing_device = $c->db_devices->active->find($c->stash('device_id'));
 
 	if ($existing_device
+		# Note! currently comparing reports *without* ignoring time series data.
 		and $existing_device->latest_report_matches($raw_report)) {
 
 		$existing_device->self_rs->latest_device_report->update({
diff --git a/lib/Conch/DB/Result/Device.pm b/lib/Conch/DB/Result/Device.pm
index 6c4bc66ae..24f5ecc2f 100644
--- a/lib/Conch/DB/Result/Device.pm
+++ b/lib/Conch/DB/Result/Device.pm
@@ -430,16 +430,22 @@ sub latest_report_data {
 Checks if the latest report's json matches the passed-in json-encoded content (comparing using
 native jsonb operators).
 
+Optionally ignores time-series data points.
+
 =cut
 
 sub latest_report_matches {
-    my ($self, $jsonb) = @_;
+    my ($self, $jsonb, $ignore_tsdb) = @_;
 
-    $self->self_rs
+    my $rs = $self->self_rs
         ->latest_device_report
-        ->as_subselect_rs
-        ->search({ report => \[ '= ?::jsonb', $jsonb ] })
-        ->exists;
+        ->as_subselect_rs;
+
+    $rs = $ignore_tsdb
+        ? $rs->matches($jsonb)
+        : $rs->search({ report => \[ '= ?::jsonb', $jsonb ] });
+
+    return $rs->exists;
 }
 
 1;
diff --git a/lib/Conch/DB/ResultSet/DeviceReport.pm b/lib/Conch/DB/ResultSet/DeviceReport.pm
new file mode 100644
index 000000000..935350f67
--- /dev/null
+++ b/lib/Conch/DB/ResultSet/DeviceReport.pm
@@ -0,0 +1,58 @@
+package Conch::DB::ResultSet::DeviceReport;
+use v5.26;
+use warnings;
+use parent 'Conch::DB::ResultSet';
+
+use Mojo::JSON 'from_json';
+
+=head1 NAME
+
+Conch::DB::ResultSet::DeviceReport
+
+=head1 DESCRIPTION
+
+Interface to queries involving device reports.
+
+=head1 METHODS
+
+=head2 matches
+
+Search for reports that match the the passed-in json blob.
+
+Current fields ignored in the comparisons:
+
+    * report_id
+    * fans
+    * psus
+    * lldp_neighbors
+    * temp
+    * disks->*->temp
+
+=cut
+
+sub matches {
+    my ($self, $jsonb) = @_;
+
+    my @disks = keys from_json($jsonb)->{disks}->%*;
+    my $ignore_fields = join(' - ', map { "'$_'" } qw(report_id fans psus lldp neighbors temp))
+        . join(' ', map { "#- '{disks,$_,temp}'" } @disks);
+
+    my $me = $self->current_source_alias;
+    $self->search(\[ "($me.report - $ignore_fields) = (?::jsonb - $ignore_fields)", $jsonb ]);
+}
+
+1;
+__END__
+
+=pod
+
+=head1 LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.
+
+=cut
+# vim: set ts=4 sts=4 sw=4 et :
diff --git a/t/integration/04_test_datacenter_loaded.t b/t/integration/04_test_datacenter_loaded.t
index c403074d8..b36bf7b8a 100644
--- a/t/integration/04_test_datacenter_loaded.t
+++ b/t/integration/04_test_datacenter_loaded.t
@@ -192,7 +192,7 @@ subtest 'Device Report' => sub {
 	is($device->related_resultset('device_relay_connections')->count, 1, 'one device_relay_connection row created');
 
 	my $dupe_report = to_json(from_json($good_report));
-	isnt($good_report, $dupe_report, 're-encoded report is not string-identical (whitespace was removed)');
+	isnt($dupe_report, $good_report, 're-encoded report is not string-identical (whitespace was removed)');
 
 	$t->post_ok('/device/TEST', { 'Content-Type' => 'application/json' }, $dupe_report)
 		->status_is(200)
@@ -286,6 +286,31 @@ subtest 'Device Report' => sub {
 		->json_is('/health' => 'PASS')
 		->json_is('/latest_report_is_invalid' => JSON::PP::false);
 
+
+TODO: {
+	local $TODO = 'not currently ignoring time series data when checking for report equivalence';
+
+	# now alter the device report so temperature is different and submit again.
+	my $report_data = from_json($good_report);
+	$report_data->{temp}{cpu0} += 100;
+	$report_data->{disks}{ (keys $report_data->{disks}->%*)[0] }{temp} += 100;
+
+	$t->post_ok('/device/TEST', json => $report_data)
+		->status_is(200)
+		->json_schema_is('ValidationState')
+		->json_is('', $validation_state_response, 'duplicate report detected, older state returned');
+
+	is($device->related_resultset('device_reports')->count, 1, 'still just one device_report row');
+	is($device->related_resultset('validation_states')->count, 1, 'still just one validation_state row');
+	is($device->related_resultset('device_relay_connections')->count, 1, 'still just one device_relay_connection');
+
+	is(
+		$device->related_resultset('device_reports')->rows(1)->get_column('received_count')->single,
+		3,
+		'received_count is incremented',
+	);
+};
+
 	cmp_deeply(
 		[ $t->app->db_devices->devices_without_location->get_column('id')->all ],
 		[ 'TEST' ],

From 848d83c81c8515c48a7fe906bd0f896012513fca Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Tue, 16 Oct 2018 14:24:52 -0700
Subject: [PATCH 2/6] cascade deletes as needed

---
 lib/Conch/DB/Result/ValidationState.pm            |  6 +++---
 lib/Conch/DB/Result/ValidationStateMember.pm      |  6 +++---
 .../0067-device_report-cascade-delete.sql         | 15 +++++++++++++++
 sql/schema.sql                                    |  4 ++--
 4 files changed, 23 insertions(+), 8 deletions(-)
 create mode 100644 sql/migrations/0067-device_report-cascade-delete.sql

diff --git a/lib/Conch/DB/Result/ValidationState.pm b/lib/Conch/DB/Result/ValidationState.pm
index e53a5ccb9..c3b3ea5ad 100644
--- a/lib/Conch/DB/Result/ValidationState.pm
+++ b/lib/Conch/DB/Result/ValidationState.pm
@@ -152,7 +152,7 @@ __PACKAGE__->belongs_to(
   "device_report",
   "Conch::DB::Result::DeviceReport",
   { id => "device_report_id" },
-  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
+  { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
 );
 
 =head2 validation_plan
@@ -200,8 +200,8 @@ __PACKAGE__->many_to_many(
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2018-10-10 16:06:16
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:gkC6RKtvMTPS5Y1V8IlugA
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2018-10-16 13:17:28
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:igN8sWgHS98xZZsbqBdZNQ
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration
diff --git a/lib/Conch/DB/Result/ValidationStateMember.pm b/lib/Conch/DB/Result/ValidationStateMember.pm
index 002adbd32..1d830352b 100644
--- a/lib/Conch/DB/Result/ValidationStateMember.pm
+++ b/lib/Conch/DB/Result/ValidationStateMember.pm
@@ -94,12 +94,12 @@ __PACKAGE__->belongs_to(
   "validation_state",
   "Conch::DB::Result::ValidationState",
   { id => "validation_state_id" },
-  { is_deferrable => 0, on_delete => "NO ACTION", on_update => "NO ACTION" },
+  { is_deferrable => 0, on_delete => "CASCADE", on_update => "NO ACTION" },
 );
 
 
-# Created by DBIx::Class::Schema::Loader v0.07049 @ 2018-09-17 14:52:33
-# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:3aEz5aEgtuv96u7AJIl+Lg
+# Created by DBIx::Class::Schema::Loader v0.07049 @ 2018-10-16 13:17:28
+# DO NOT MODIFY THIS OR ANYTHING ABOVE! md5sum:waeVK0lJGoJZbkd640IX7A
 
 
 # You can replace this text with custom code or comments, and it will be preserved on regeneration
diff --git a/sql/migrations/0067-device_report-cascade-delete.sql b/sql/migrations/0067-device_report-cascade-delete.sql
new file mode 100644
index 000000000..b9dd5f6c5
--- /dev/null
+++ b/sql/migrations/0067-device_report-cascade-delete.sql
@@ -0,0 +1,15 @@
+SELECT run_migration(67, $$
+
+    -- when device_report is deleted, delete validation_state records that point to it
+    alter table validation_state
+        drop constraint validation_state_device_report_id_fkey,
+        add constraint validation_state_device_report_id_fkey
+            foreign key (device_report_id) references device_report(id) on delete cascade;
+
+    -- when validation_state is deleted, delete validation_state_member records that point to it
+    alter table validation_state_member
+        drop constraint validation_state_member_validation_state_id_fkey,
+        add constraint validation_state_member_validation_state_id_fkey
+            foreign key (validation_state_id) references validation_state(id) on delete cascade;
+
+$$);
diff --git a/sql/schema.sql b/sql/schema.sql
index cf702fe66..4de685ccf 100644
--- a/sql/schema.sql
+++ b/sql/schema.sql
@@ -1945,7 +1945,7 @@ ALTER TABLE ONLY public.validation_state
 --
 
 ALTER TABLE ONLY public.validation_state
-    ADD CONSTRAINT validation_state_device_report_id_fkey FOREIGN KEY (device_report_id) REFERENCES public.device_report(id);
+    ADD CONSTRAINT validation_state_device_report_id_fkey FOREIGN KEY (device_report_id) REFERENCES public.device_report(id) ON DELETE CASCADE;
 
 
 --
@@ -1961,7 +1961,7 @@ ALTER TABLE ONLY public.validation_state_member
 --
 
 ALTER TABLE ONLY public.validation_state_member
-    ADD CONSTRAINT validation_state_member_validation_state_id_fkey FOREIGN KEY (validation_state_id) REFERENCES public.validation_state(id);
+    ADD CONSTRAINT validation_state_member_validation_state_id_fkey FOREIGN KEY (validation_state_id) REFERENCES public.validation_state(id) ON DELETE CASCADE;
 
 
 --

From fcad3d2251a587ed54d10f96934665a6dfd21cdf Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Thu, 18 Oct 2018 09:26:01 -0700
Subject: [PATCH 3/6] add $rs->page helper

---
 lib/Conch/DB/ResultSet.pm | 1 +
 1 file changed, 1 insertion(+)

diff --git a/lib/Conch/DB/ResultSet.pm b/lib/Conch/DB/ResultSet.pm
index 229e4fed3..ecc2b0298 100644
--- a/lib/Conch/DB/ResultSet.pm
+++ b/lib/Conch/DB/ResultSet.pm
@@ -24,6 +24,7 @@ __PACKAGE__->load_components(
     'Helper::ResultSet::Shortcut::Distinct',    # provides distinct
     '+Conch::DB::ResultsExist',                 # provides exists
     'Helper::ResultSet::Shortcut::Columns',     # provides columns
+    'Helper::ResultSet::Shortcut::Page',        # provides page
 );
 
 1;

From 4d0439124602c28cd99f345fe5e33ac76e48e743 Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Tue, 16 Oct 2018 11:24:58 -0700
Subject: [PATCH 4/6] script to remove duplicate device reports

---
 lib/Conch/Command/dedupe_device_reports.pm | 180 +++++++++++++++++++++
 lib/Conch/DB/Result/Device.pm              |   2 +-
 lib/Conch/DB/ResultSet/DeviceReport.pm     |  49 +++++-
 3 files changed, 228 insertions(+), 3 deletions(-)
 create mode 100644 lib/Conch/Command/dedupe_device_reports.pm

diff --git a/lib/Conch/Command/dedupe_device_reports.pm b/lib/Conch/Command/dedupe_device_reports.pm
new file mode 100644
index 000000000..a247c3d50
--- /dev/null
+++ b/lib/Conch/Command/dedupe_device_reports.pm
@@ -0,0 +1,180 @@
+package Conch::Command::dedupe_device_reports;
+
+=pod
+
+=head1 NAME
+
+dedupe_device_reports - remove duplicate device reports
+
+=head1 SYNOPSIS
+
+    dedupe_device_reports [long options...]
+
+        --help  print usage message and exit
+
+=cut
+
+use Mojo::Base 'Mojolicious::Command', -signatures;
+use Getopt::Long::Descriptive;
+use Try::Tiny;
+
+has description => 'remove duplicate device reports';
+
+has usage => sub { shift->extract_usage };  # extracts from SYNOPSIS
+
+has 'dry_run';
+
+sub run {
+    my $self = shift;
+
+    # if the user needs to ^C, print the post-processing statistics before exiting.
+    local $SIG{INT} = sub {
+        say "\naborting! We now have this many records:";
+        $self->_print_stats;
+        exit;
+    };
+
+    local @ARGV = @_;
+    my ($opt, $usage) = describe_options(
+        # the descriptions aren't actually used anymore (mojo uses the synopsis instead)... but
+        # the 'usage' text block can be accessed with $usage->text
+        'dedupe_device_reports %o',
+        [ 'dry-run|n',      'dry-run (no changes are made)' ],
+        [],
+        [ 'help',           'print usage message and exit', { shortcircuit => 1 } ],
+    );
+
+    $self->dry_run($opt->dry_run);
+
+    say 'at start, we have this many records:';
+    $self->_print_stats;
+
+    # consider each device, oldest devices first, in pages of 100 rows each
+    my $device_rs = ($self->dry_run ? $self->app->db_ro_devices : $self->app->db_devices)
+        ->active
+        ->rows(100)
+        ->page(1)
+        ->order_by('created');
+
+    my $device_count = 0;
+
+    foreach my $page (1 .. $device_rs->pager->last_page) {
+
+        $device_rs = $device_rs->page($page);
+        while (my $device = $device_rs->next) {
+
+            # we process each device's reports in a separate transaction,
+            # so we can abort and resume without redoing everything all over again
+            try {
+                $self->app->schema->txn_do(sub {
+                    $self->_process_device($device);
+                });
+                ++$device_count;
+            }
+            catch {
+                if ($_ =~ /Rollback failed/) {
+                    local $@ = $_;
+                    die;    # propagate the error
+                }
+                print STDERR "\n", 'aborted processing of device ' . $device->id . ': ', $_, "\n";
+            };
+        }
+    }
+
+    say "\n$device_count devices processed.";
+
+    say 'at finish, we have this many records:';
+    $self->_print_stats;
+}
+
+sub _print_stats ($self) {
+    say 'device_report:           ', $self->app->db_ro_device_reports->count;
+    say 'validation_state:        ', $self->app->db_ro_validation_states->count;
+    say 'validation_state_member: ', $self->app->db_ro_validation_state_members->count;
+    say 'validation_result:       ', $self->app->db_ro_validation_results->count;
+}
+
+sub _process_device ($self, $device) {
+
+    my $report_count = 0;
+    print 'device id ', $device->id, ': ';
+
+    # consider all PASSING device reports, newest first, in pages of 100 rows each
+    my $device_report_rs = $device
+        ->search_related('validation_states', { status => 'pass' })
+        ->related_resultset('device_report')
+        ->columns([ qw(id created) ])
+        ->rows(100)
+        ->page(1)
+        ->order_by({ -desc => 'created' })
+        ->hri;  # raw hashref data; do not inflate to objects or alter timestamps
+
+    # we accumulate report ids to delete so we can safely iterate through reports
+    # without the pages changing strangely
+    my @delete_report_ids;
+
+    foreach my $page (1 .. $device_report_rs->pager->last_page) {
+        print "\n" if $page % 100 == 0;
+        print '.';
+
+        $device_report_rs = $device_report_rs->page($page);
+
+        while (my $device_report = $device_report_rs->next) {
+            ++$report_count;
+            print '.' if $page % 100 == 0;
+
+            # delete this report if it is identical (excluding time-series data)
+            # to another report. (a *newer* report may be found if it did not have a
+            # validation_state record linked to it, but usually the matching duplicate will be
+            # older.)
+            if ($device->related_resultset('device_reports')
+                ->matches_report_id($device_report->{id})
+                ->exists)
+            {
+                print 'x';
+                push @delete_report_ids, $device_report->{id};
+            }
+        }
+    }
+
+    print "\n";
+
+    if ($self->dry_run) {
+        say 'Would delete ', scalar(@delete_report_ids), ' reports for device id ', $device->id,
+            ' out of ', $report_count, ' examined.';
+    }
+    else {
+        # delete all duplicate reports that we found
+        # this may also cause cascade deletes on validation_state, validation_state_member.
+        say 'deleting ', scalar(@delete_report_ids), ' reports for device id ', $device->id,
+            ' out of ', $report_count, ' examined...';
+        $device
+            ->search_related('device_reports', { id => { -in => \@delete_report_ids } })
+            ->delete;
+
+        # delete all newly-orphaned validation_result rows for this device
+        $device
+            ->search_related('validation_results',
+                { 'validation_state_members.validation_result_id' => undef },
+                { join => 'validation_state_members' },
+            )->delete;
+    }
+
+    print "\n";
+}
+
+1;
+__END__
+
+=pod
+
+=head1 LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.
+
+=cut
+# vim: set ts=4 sts=4 sw=4 et :
diff --git a/lib/Conch/DB/Result/Device.pm b/lib/Conch/DB/Result/Device.pm
index 24f5ecc2f..ea94fa794 100644
--- a/lib/Conch/DB/Result/Device.pm
+++ b/lib/Conch/DB/Result/Device.pm
@@ -442,7 +442,7 @@ sub latest_report_matches {
         ->as_subselect_rs;
 
     $rs = $ignore_tsdb
-        ? $rs->matches($jsonb)
+        ? $rs->matches_jsonb($jsonb)
         : $rs->search({ report => \[ '= ?::jsonb', $jsonb ] });
 
     return $rs->exists;
diff --git a/lib/Conch/DB/ResultSet/DeviceReport.pm b/lib/Conch/DB/ResultSet/DeviceReport.pm
index 935350f67..495b8231f 100644
--- a/lib/Conch/DB/ResultSet/DeviceReport.pm
+++ b/lib/Conch/DB/ResultSet/DeviceReport.pm
@@ -4,6 +4,7 @@ use warnings;
 use parent 'Conch::DB::ResultSet';
 
 use Mojo::JSON 'from_json';
+use Conch::UUID 'is_uuid';
 
 =head1 NAME
 
@@ -15,7 +16,7 @@ Interface to queries involving device reports.
 
 =head1 METHODS
 
-=head2 matches
+=head2 matches_jsonb
 
 Search for reports that match the the passed-in json blob.
 
@@ -30,7 +31,7 @@ Current fields ignored in the comparisons:
 
 =cut
 
-sub matches {
+sub matches_jsonb {
     my ($self, $jsonb) = @_;
 
     my @disks = keys from_json($jsonb)->{disks}->%*;
@@ -41,6 +42,50 @@ sub matches {
     $self->search(\[ "($me.report - $ignore_fields) = (?::jsonb - $ignore_fields)", $jsonb ]);
 }
 
+=head2 matches_report_id
+
+Search for reports that match the json blob from the report referenced by the passed-in id.
+
+Current fields ignored in the comparisons:
+
+    * report_id
+    * fans
+    * psus
+    * lldp_neighbors
+    * temp
+    * disks->*->temp
+
+=cut
+
+sub matches_report_id {
+    my ($self, $report_id) = @_;
+
+    Carp::croak('did not supply report_id') if not $report_id or not is_uuid($report_id);
+
+    my $compare_report_rs = $self->result_source->resultset
+        ->search({ 'subquery.id' => $report_id }, { alias => 'subquery' });
+
+    # ideally I'd like to do all this server-side, but I'm not sure how to subtract
+    # all these from 'device_report.report' once I've assembled:
+    # select '{disks,' || disk || ',temp}'
+    # from (select jsonb_object_keys(report->'disks') as disk from device_report where id = ?
+
+    my @disks = $compare_report_rs
+        ->search(undef, { select => \q{jsonb_object_keys(subquery.report->'disks')}, as => 'disk' })
+        ->get_column('disk')
+        ->all;
+
+    my $ignore_fields = join(' - ', map { "'$_'" } qw(report_id fans psus lldp neighbors temp))
+        . join(' ', map { "#- '{disks,$_,temp}'" } @disks);
+
+    my $me = $self->current_source_alias;
+
+    my $jsonb_rs = $compare_report_rs
+        ->search(undef, { select => \"report - $ignore_fields" });
+
+    $self->search({ "($me.report - $ignore_fields)" => { '=' => $jsonb_rs->as_query } });
+}
+
 1;
 __END__
 

From 50cd5ff26287196500500a9d7b49342a5f758ff6 Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Wed, 17 Oct 2018 16:38:09 -0700
Subject: [PATCH 5/6] script for extracting all temperatures from historical
 device reports

see #458.
---
 lib/Conch/Command/extract_temperatures.pm | 138 ++++++++++++++++++++++
 1 file changed, 138 insertions(+)
 create mode 100644 lib/Conch/Command/extract_temperatures.pm

diff --git a/lib/Conch/Command/extract_temperatures.pm b/lib/Conch/Command/extract_temperatures.pm
new file mode 100644
index 000000000..66999ae47
--- /dev/null
+++ b/lib/Conch/Command/extract_temperatures.pm
@@ -0,0 +1,138 @@
+package Conch::Command::extract_temperatures;
+
+=pod
+
+=head1 NAME
+
+extract_temperatures - extract temperatures from historical device reports
+
+=head1 SYNOPSIS
+
+    extract_temperatures [long options...]
+
+        --dir   directory to create data files in
+        --help  print usage message and exit
+
+=cut
+
+use Mojo::Base 'Mojolicious::Command', -signatures;
+use Getopt::Long::Descriptive;
+use Mojo::JSON 'from_json', 'encode_json';
+
+has description => 'extract temperatures from historical device reports';
+
+has usage => sub { shift->extract_usage };  # extracts from SYNOPSIS
+
+sub run {
+    my $self = shift;
+
+    local @ARGV = @_;
+    my ($opt, $usage) = describe_options(
+        # the descriptions aren't actually used anymore (mojo uses the synopsis instead)... but
+        # the 'usage' text block can be accessed with $usage->text
+        'extract_temperatures %o',
+        [ 'dir|d=s',        'directory to create data files in' ],
+        [],
+        [ 'help',           'print usage message and exit', { shortcircuit => 1 } ],
+    );
+
+    if ($opt->dir) {
+        say 'creating files in ', $opt->dir, '...';
+        mkdir $opt->dir if not -d $opt->dir;
+        chdir $opt->dir;
+    }
+
+    # process reports in pages of 100 rows each
+    my $device_report_rs = $self->app->db_ro_device_reports
+        ->rows(100)
+        ->page(1)
+        ->hri;  # raw hashref data; do not inflate to objects or alter timestamps
+
+    my $num_pages = $device_report_rs->pager->last_page;
+    foreach my $page (1 .. $num_pages) {
+        print "\n" if $page % 100 == 0;
+        print '.';
+
+        $device_report_rs = $device_report_rs->page($page);
+        while (my $device_report = $device_report_rs->next) {
+
+            my $data = from_json($device_report->{report});
+
+            # TODO: I have no idea if this is the desired format! adjust as needed.
+
+            my $temp = +{
+                $data->{temp}->%*,
+
+                (map {; "disk_$_" => $data->{disks}{$_}{temp} }
+                    grep { exists $data->{disks}{$_}{temp} }
+                    keys $data->{disks}->%*),
+            };
+
+            my $fan_speeds = +{
+                (map {; "fan_$_" => $data->{fans}{units}[$_]{speed_pct} }
+                    grep { exists $data->{fans}{units}[$_]{speed_pct} }
+                    0 .. $data->{fans}{count} - 1),
+            };
+
+            my $psus = +{
+                (map {;
+                    my $num = $_;
+                    "psu_$num" => +{
+                        map { $_ => $data->{psus}{units}[$num]{$_} }
+                            grep { /^(amps|volts|watts)/ }
+                            keys $data->{psus}{units}[$num]->%*
+                    }
+                }
+                0 .. $data->{psus}{count} - 1),
+            };
+
+            my $output_data = {
+                date => $device_report->{created},
+                device_id => $device_report->{device_id},
+                ( keys %$temp ? ( temp => $temp ) : () ),
+                ( keys %$fan_speeds ? ( fan_speeds => $fan_speeds ) : () ),
+                ( keys %$psus ? ( psus => $psus ) : () ),
+            };
+
+            next if keys %$output_data == 2;
+
+            my $fh = $self->_fh_for_date($device_report->{created});
+            print $fh encode_json($output_data), "\n";
+        }
+    }
+
+    print "\n\ndone.\n";
+}
+
+my %fh_cache;
+
+sub _fh_for_date ($self, $timestamp) {
+
+    my $date = Conch::Time->new($timestamp)->strftime('%Y-%m-%d');
+
+    return $fh_cache{$date} if exists $fh_cache{$date};
+
+    # we're on a new date; close the old file and open a new one
+    close $_ foreach values %fh_cache;
+
+    # use raw binmode, as json data will be utf8-encoded if needed.
+    open $fh_cache{$date}, '>', "temperatures-$date.json"
+        or die "could not open temperatures-$date.json for writing: $!";
+    return $fh_cache{$date};
+}
+
+1;
+__END__
+
+=pod
+
+=head1 LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.
+
+=cut
+# vim: set ts=4 sts=4 sw=4 et :

From b0ea120107a03b89c509e0cc12b4b1298dd3f19b Mon Sep 17 00:00:00 2001
From: Karen Etheridge <ether@joyent.com>
Date: Fri, 19 Oct 2018 12:27:18 -0700
Subject: [PATCH 6/6] wip for /device/:id/environment endpoint

---
 json-schema/input.yaml                    | 34 ++++++++-
 lib/Conch/Controller/DeviceEnvironment.pm | 90 +++++++++++++++++++++++
 lib/Conch/Controller/DeviceReport.pm      |  3 +-
 lib/Conch/Route/Device.pm                 |  5 ++
 4 files changed, 128 insertions(+), 4 deletions(-)
 create mode 100644 lib/Conch/Controller/DeviceEnvironment.pm

diff --git a/json-schema/input.yaml b/json-schema/input.yaml
index 62436cb54..12de2c166 100644
--- a/json-schema/input.yaml
+++ b/json-schema/input.yaml
@@ -84,7 +84,35 @@ definitions:
         type: string
       vendor_name:
         type: string
-
+  EnvironmentData:
+    properties:
+      temp:
+        type: object
+        required:
+          - cpu0
+          - cpu1
+        properties:
+          cpu0:
+            type: integer
+          cpu1:
+            type: integer
+          exhaust:
+            type: integer
+          inlet:
+            type: integer
+          disks:
+            type: object
+            patternProperties:
+              ^\S+$:
+                description: key = device_disk.serial_number
+                type: integer
+      voltage:
+        type: object
+        properties:
+          psu0:
+            type: number
+          psu1:
+            type: number
   DeviceReport:
     required:
       - bios_version
@@ -121,7 +149,7 @@ definitions:
               drive_type:
                 type: string
               temp:
-                # TODO: move to different endpoint
+                # TODO: remove, once /device/:id/environment endpoint starts getting used
                 $ref: /definitions/int_or_stringy_int
               enclosure:
                 type: string
@@ -220,7 +248,7 @@ definitions:
       system_uuid:
         $ref: /definitions/uuid
       temp:
-        # TODO: move to different endpoint
+        # TODO: remove, once /device/:id/environment endpoint starts getting used
         type: object
         required:
           - cpu0
diff --git a/lib/Conch/Controller/DeviceEnvironment.pm b/lib/Conch/Controller/DeviceEnvironment.pm
new file mode 100644
index 000000000..9538e3364
--- /dev/null
+++ b/lib/Conch/Controller/DeviceEnvironment.pm
@@ -0,0 +1,90 @@
+package Conch::Controller::DeviceEnvironment;
+
+use Mojo::Base 'Mojolicious::Controller', -signatures;
+
+use Role::Tiny::With;
+with 'Conch::Role::MojoLog';
+
+=pod
+
+=head1 NAME
+
+Conch::Controller::DeviceEnvironment
+
+=head1 METHODS
+
+=head2 process
+
+Receives environment data for a particular device:
+
+- records to the database
+- dispatches to Circonus/Prometheus (TODO)
+- sends to a validation (TODO) (what to do when validation fails?)
+
+=cut
+
+sub process ($c) {
+    my $data = $c->validate_input('EnvironmentData');
+    return if not $data;
+
+    my $device_rs = $c->stash('device_rs');
+
+    my %environment = (
+        $data->{temp} ? (
+            cpu0_temp    => $data->{temp}{cpu0},
+            cpu1_temp    => $data->{temp}{cpu1},
+            inlet_temp   => $data->{temp}{inlet},
+            exhaust_temp => $data->{temp}{exhaust},
+        ) : (),
+        $data->{voltage} ? (
+            psu0_voltage => $data->{voltage}{psu0},
+            psu1_voltage => $data->{voltage}{psu1},
+        ) : (),
+    );
+
+    $device_rs->related_resultset('device_environment')->update_or_create({
+        %environment,
+        updated => \'now()',
+    }) if keys %environment;
+
+    if ($data->{disks} and keys $data->{disks}->%*) {
+        foreach my $disk_serial (keys $data->{disks}->%*) {
+            my $disk = $device_rs->related_resultset('device_disks')->find(
+                { serial_number => $disk_serial },
+                { key => 'device_disk_serial_number_key' },
+            );
+            if (not $disk) {
+                $c->log->debug('received environment data for non-existent disk: device id '
+                    .$c->stash('device_id').", serial number $disk_serial");
+                next;
+            }
+
+            $disk->update({
+                temp => $data->{disks}{$disk_serial}{temp},
+                updated => \'now()',
+            });
+        }
+    }
+
+    $c->log->info('recorded environment data for device '.$c->stash('device_id'));
+
+    # TODO: send to Circonus/Prometheus.
+
+    # TODO: run validations?
+}
+
+1;
+__END__
+
+=pod
+
+=head1 LICENSING
+
+Copyright Joyent, Inc.
+
+This Source Code Form is subject to the terms of the Mozilla Public License,
+v.2.0. If a copy of the MPL was not distributed with this file, You can obtain
+one at http://mozilla.org/MPL/2.0/.
+
+=cut
+# vim: set ts=4 sts=4 sw=4 et :
diff --git a/lib/Conch/Controller/DeviceReport.pm b/lib/Conch/Controller/DeviceReport.pm
index 54095d19c..db12c5f70 100644
--- a/lib/Conch/Controller/DeviceReport.pm
+++ b/lib/Conch/Controller/DeviceReport.pm
@@ -255,13 +255,14 @@ sub _record_device_configuration {
 
 			$log->info("Created Device Spec for Device ".$device->id);
 
+			# TODO: stop doing this, and send all temp data to /device/:id/environment instead.
 			if ($dr->{temp}) {
 				$device->related_resultset('device_environment')->update_or_create({
 					cpu0_temp    => $dr->{temp}->{cpu0},
 					cpu1_temp    => $dr->{temp}->{cpu1},
 					inlet_temp   => $dr->{temp}->{inlet},
 					exhaust_temp => $dr->{temp}->{exhaust},
-					# TODO: not setting psu0_voltage, psu1_voltage
+					# note: not setting psu0_voltage, psu1_voltage
 					updated      => \'NOW()',
 				});
 				$c->log->info("Recorded environment for Device ".$device->id);
diff --git a/lib/Conch/Route/Device.pm b/lib/Conch/Route/Device.pm
index e90c193e4..18ebe17bd 100644
--- a/lib/Conch/Route/Device.pm
+++ b/lib/Conch/Route/Device.pm
@@ -43,6 +43,8 @@ Sets up the routes for /device:
     GET     /device/:device_id/interface/:interface_name
     GET     /device/:device_id/interface/:interface_name/:field
 
+    POST    /device/:device_id/environment
+
 =cut
 
 sub routes {
@@ -133,6 +135,9 @@ sub routes {
             # GET /device/:device_id/interface/:interface_name/:field
             $with_interface_name->get('/:field_name')->to('#get_one_field');
         }
+
+        # POST /device/:device_id/environment
+        $with_device->post('/environment')->to('device_environment#process');
     }
 }