Skip to content

Commit

Permalink
Support missing samples in oximeter
Browse files Browse the repository at this point in the history 
- Add a `Datum::Missing` and `MissingDatum`, which records the intended
  datum type and an optional start time for a sample which could not be
  produced.
- Database upgrades which make all scalar datum columns Nullable. Array
  fields are _not_ made Nullable, since ClickHouse doesn't support
  composite types like arrays inside a Nullable wrapper type. The empty
  array is used as a sentinel, which is OK since we can't have
  zero-length array histograms in Oximeter. Add a test which will fail
  if we ever change that.
- Rework database serialization to handle Nullable types or empty
  arrays. This uses a new helper trait to convert a NULL (which has no
  type information) to the intended datum type, or an empty array to a
  histogram.
- Add a test for each measurement type that we can recover a missing
  sample of that type -- NULLs for scalar values and empty arrays for
  histograms.
  • Loading branch information
@bnaecker
bnaecker committed Nov 29, 2023
1 parent bb7ee84 commit 8b5abc4
Show file tree
Hide file tree
Showing 62 changed files with 1,139 additions and 466 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

119 changes: 119 additions & 0 deletions openapi/nexus-internal.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2568,9 +2568,60 @@
"datum",
"type"
]
},
{
"type": "object",
"properties": {
"datum": {
"$ref": "#/components/schemas/MissingDatum"
},
"type": {
"type": "string",
"enum": [
"missing"
]
}
},
"required": [
"datum",
"type"
]
}
]
},
"DatumType": {
"description": "The type of an individual datum of a metric.",
"type": "string",
"enum": [
"bool",
"i8",
"u8",
"i16",
"u16",
"i32",
"u32",
"i64",
"u64",
"f32",
"f64",
"string",
"bytes",
"cumulative_i64",
"cumulative_u64",
"cumulative_f32",
"cumulative_f64",
"histogram_i8",
"histogram_u8",
"histogram_i16",
"histogram_u16",
"histogram_i32",
"histogram_u32",
"histogram_i64",
"histogram_u64",
"histogram_f32",
"histogram_f64"
]
},
"DiskRuntimeState": {
"description": "Runtime state of the Disk, which includes its attach state and some minimal metadata",
"type": "object",
Expand Down Expand Up @@ -4128,9 +4179,77 @@
"content",
"type"
]
},
{
"type": "object",
"properties": {
"content": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
}
},
"required": [
"datum_type"
]
},
"type": {
"type": "string",
"enum": [
"missing_datum_requires_start_time"
]
}
},
"required": [
"content",
"type"
]
},
{
"type": "object",
"properties": {
"content": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
}
},
"required": [
"datum_type"
]
},
"type": {
"type": "string",
"enum": [
"missing_datum_cannot_have_start_time"
]
}
},
"required": [
"content",
"type"
]
}
]
},
"MissingDatum": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
},
"start_time": {
"nullable": true,
"type": "string",
"format": "date-time"
}
},
"required": [
"datum_type"
]
},
"Name": {
"title": "A name unique within the parent collection",
"description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.",
Expand Down
67 changes: 67 additions & 0 deletions openapi/nexus.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9728,9 +9728,60 @@
"datum",
"type"
]
},
{
"type": "object",
"properties": {
"datum": {
"$ref": "#/components/schemas/MissingDatum"
},
"type": {
"type": "string",
"enum": [
"missing"
]
}
},
"required": [
"datum",
"type"
]
}
]
},
"DatumType": {
"description": "The type of an individual datum of a metric.",
"type": "string",
"enum": [
"bool",
"i8",
"u8",
"i16",
"u16",
"i32",
"u32",
"i64",
"u64",
"f32",
"f64",
"string",
"bytes",
"cumulative_i64",
"cumulative_u64",
"cumulative_f32",
"cumulative_f64",
"histogram_i8",
"histogram_u8",
"histogram_i16",
"histogram_u16",
"histogram_i32",
"histogram_u32",
"histogram_i64",
"histogram_u64",
"histogram_f32",
"histogram_f64"
]
},
"DerEncodedKeyPair": {
"type": "object",
"properties": {
Expand Down Expand Up @@ -12250,6 +12301,22 @@
"items"
]
},
"MissingDatum": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
},
"start_time": {
"nullable": true,
"type": "string",
"format": "date-time"
}
},
"required": [
"datum_type"
]
},
"Name": {
"title": "A name unique within the parent collection",
"description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.",
Expand Down
119 changes: 119 additions & 0 deletions openapi/sled-agent.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3002,9 +3002,60 @@
"datum",
"type"
]
},
{
"type": "object",
"properties": {
"datum": {
"$ref": "#/components/schemas/MissingDatum"
},
"type": {
"type": "string",
"enum": [
"missing"
]
}
},
"required": [
"datum",
"type"
]
}
]
},
"DatumType": {
"description": "The type of an individual datum of a metric.",
"type": "string",
"enum": [
"bool",
"i8",
"u8",
"i16",
"u16",
"i32",
"u32",
"i64",
"u64",
"f32",
"f64",
"string",
"bytes",
"cumulative_i64",
"cumulative_u64",
"cumulative_f32",
"cumulative_f64",
"histogram_i8",
"histogram_u8",
"histogram_i16",
"histogram_u16",
"histogram_i32",
"histogram_u32",
"histogram_i64",
"histogram_u64",
"histogram_f32",
"histogram_f64"
]
},
"DeleteVirtualNetworkInterfaceHost": {
"description": "The data needed to identify a virtual IP for which a sled maintains an OPTE virtual-to-physical mapping such that that mapping can be deleted.",
"type": "object",
Expand Down Expand Up @@ -4923,9 +4974,77 @@
"content",
"type"
]
},
{
"type": "object",
"properties": {
"content": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
}
},
"required": [
"datum_type"
]
},
"type": {
"type": "string",
"enum": [
"missing_datum_requires_start_time"
]
}
},
"required": [
"content",
"type"
]
},
{
"type": "object",
"properties": {
"content": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
}
},
"required": [
"datum_type"
]
},
"type": {
"type": "string",
"enum": [
"missing_datum_cannot_have_start_time"
]
}
},
"required": [
"content",
"type"
]
}
]
},
"MissingDatum": {
"type": "object",
"properties": {
"datum_type": {
"$ref": "#/components/schemas/DatumType"
},
"start_time": {
"nullable": true,
"type": "string",
"format": "date-time"
}
},
"required": [
"datum_type"
]
},
"Name": {
"title": "A name unique within the parent collection",
"description": "Names must begin with a lower case ASCII letter, be composed exclusively of lowercase ASCII, uppercase ASCII, numbers, and '-', and may not end with a '-'. Names cannot be a UUID though they may contain a UUID.",
Expand Down
Loading

0 comments on commit 8b5abc4

Please sign in to comment.