Skip to content

Commit

Permalink
feat: update sqlite to use a hand build schema
Browse files Browse the repository at this point in the history 
This is the next iteration on from the basic table layout, there has
been more effort in normalisation.  There are still some outstanding
issues, one improvement would be to consolidate some of the tables.
  • Loading branch information
@ozym
ozym committed Dec 20, 2024
1 parent 8bb1675 commit 5615c7a
Show file tree
Hide file tree
Showing 35 changed files with 6,434 additions and 1,066 deletions.
64 changes: 32 additions & 32 deletions assets/dataloggers.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -974,38 +974,38 @@ Quanterra,Q4120/6,990801,12996,Obsolete
Quanterra,Q4120/6,990802,12995,Obsolete
Quanterra,Q730/3,980307,,Obsolete
Quanterra,Q730/4,980306,,Obsolete
Reftek,130,902A,,
Reftek,130,902B,,
Reftek,130,902C,,
Reftek,130,902D,,
Reftek,130,9337,,
Reftek,130,953D,,
Reftek,130,9793,,
Reftek,130,989C,,
Reftek,130,9D3D,,
Reftek,130,9F99,,
Reftek,130,9F9A,,
Reftek,130,A895,,
Reftek,130,AC13,,
Reftek,130,AC14,,
Reftek,REFTEK-RECORDER-1995,384,,
Reftek,REFTEK-RECORDER-1995,528,,
Reftek,REFTEK-RECORDER-2001,243,,
Reftek,REFTEK-RECORDER-2001,274,,
Reftek,REFTEK-RECORDER-2001,276,,
Reftek,REFTEK-RECORDER-2001,277,,
Reftek,REFTEK-RECORDER-2001,280,,
Reftek,REFTEK-RECORDER-2001,296,,
Reftek,REFTEK-RECORDER-2001,301,,
Reftek,REFTEK-RECORDER-2001,323,,
Reftek,REFTEK-RECORDER-2001,325,,
Reftek,REFTEK-RECORDER-2001,327,,
Reftek,REFTEK-RECORDER-2001,330,,
Reftek,REFTEK-RECORDER-2001,331,,
Reftek,REFTEK-RECORDER-2001,332,,
Reftek,REFTEK-RECORDER-2001,333,,
Reftek,REFTEK-RECORDER-2001,335,,
Reftek,REFTEK-RECORDER-2001,336,,
RefTek,130,902A,,
RefTek,130,902B,,
RefTek,130,902C,,
RefTek,130,902D,,
RefTek,130,9337,,
RefTek,130,953D,,
RefTek,130,9793,,
RefTek,130,989C,,
RefTek,130,9D3D,,
RefTek,130,9F99,,
RefTek,130,9F9A,,
RefTek,130,A895,,
RefTek,130,AC13,,
RefTek,130,AC14,,
RefTek,REFTEK-RECORDER-1995,384,,
RefTek,REFTEK-RECORDER-1995,528,,
RefTek,REFTEK-RECORDER-2001,243,,
RefTek,REFTEK-RECORDER-2001,274,,
RefTek,REFTEK-RECORDER-2001,276,,
RefTek,REFTEK-RECORDER-2001,277,,
RefTek,REFTEK-RECORDER-2001,280,,
RefTek,REFTEK-RECORDER-2001,296,,
RefTek,REFTEK-RECORDER-2001,301,,
RefTek,REFTEK-RECORDER-2001,323,,
RefTek,REFTEK-RECORDER-2001,325,,
RefTek,REFTEK-RECORDER-2001,327,,
RefTek,REFTEK-RECORDER-2001,330,,
RefTek,REFTEK-RECORDER-2001,331,,
RefTek,REFTEK-RECORDER-2001,332,,
RefTek,REFTEK-RECORDER-2001,333,,
RefTek,REFTEK-RECORDER-2001,335,,
RefTek,REFTEK-RECORDER-2001,336,,
Sprengnether Instruments Inc.,SPRENGNETHER,CAZ_1,,
Unknown-logger-make,Unknown-logger-model,Unknown,,Ruapehu Crater Lake 1993-95
Wenner,WENNER,WEL_1,,
Expand Down
36 changes: 21 additions & 15 deletions cmd/deltadb/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,12 +51,14 @@ and then to examine the file
sqlite3 delta.db
SQLite version 3.46.1 2024-08-13 09:16:08
Enter ".help" for usage hints.
sqlite> .schema Network
CREATE TABLE Network (
Code TEXT PRIMARY KEY,
External TEXT,
Description TEXT,
Restricted TEXT
sqlite> .schema network
CREATE TABLE network (
network_id INTEGER PRIMARY KEY NOT NULL,
network TEXT NOT NULL,
external TEXT NOT NULL,
description TEXT DEFAULT "" NOT NULL,
restricted BOOLEAN DEFAULT false NOT NULL,
UNIQUE (network)
);
sqlite>
```
Expand Down Expand Up @@ -90,16 +92,20 @@ curl -s localhost:8080/network|jq .|head
or

```
curl -s localhost:8080/station/WEL/site/20/sensor/|jq . | head
curl -s localhost:8080/station/WEL|jq . | head
[
{
"make": "Kinemetrics",
"model": "FBA-23",
"serial": "25073",
"start": "1990-10-03T03:04:00Z",
"end": "2003-04-08T00:00:00Z",
"dip": 0,
"azimuth": 0,
"factor": 0,
"station": "WEL",
"network": "NZ",
"name": "Wellington",
"latitude": -41.284047578,
"longitude": 174.768184021,
"elevation": 138,
"datum": "WGS84",
"start": "1916-01-01T00:00:00Z",
...
```

## schema

[![Schema](delta.svg)](delta.svg)
3,703 changes: 3,703 additions & 0 deletions cmd/deltadb/delta.svg
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
30 changes: 0 additions & 30 deletions cmd/deltadb/handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,53 +19,23 @@ func newHandler(db *sql.DB) *http.ServeMux {
mux.HandleFunc("/", homePage)
mux.HandleFunc("GET /network", h.GetNetworks)
mux.HandleFunc("GET /network/", h.GetNetworks)
mux.HandleFunc("GET /network/{code}", h.GetNetworkCode)
mux.HandleFunc("GET /network/{code}/", h.GetNetworkCode)
mux.HandleFunc("GET /network/{code}/station", h.GetNetworkStations)
mux.HandleFunc("GET /network/{code}/station/", h.GetNetworkStations)
mux.HandleFunc("GET /network/{code}/mark", h.GetNetworkMarks)
mux.HandleFunc("GET /network/{code}/mark/", h.GetNetworkMarks)
mux.HandleFunc("GET /network/{code}/mount", h.GetNetworkMounts)
mux.HandleFunc("GET /network/{code}/mount/", h.GetNetworkMounts)
mux.HandleFunc("GET /network/{code}/sample", h.GetNetworkSamples)
mux.HandleFunc("GET /network/{code}/sample/", h.GetNetworkSamples)

mux.HandleFunc("GET /station", h.GetStations)
mux.HandleFunc("GET /station/", h.GetStations)
mux.HandleFunc("GET /station/{code}", h.GetStationCode)
mux.HandleFunc("GET /station/{code}/", h.GetStationCode)
mux.HandleFunc("GET /station/{code}/site", h.GetStationSites)
mux.HandleFunc("GET /station/{code}/site/", h.GetStationSites)
mux.HandleFunc("GET /station/{code}/site/{location}", h.GetStationSiteLocation)
mux.HandleFunc("GET /station/{code}/site/{location}/", h.GetStationSiteLocation)
mux.HandleFunc("GET /station/{code}/site/{location}/sensor", h.GetStationSiteSensors)
mux.HandleFunc("GET /station/{code}/site/{location}/sensor/", h.GetStationSiteSensors)

mux.HandleFunc("GET /site", h.GetSites)
mux.HandleFunc("GET /site/", h.GetSites)
mux.HandleFunc("GET /site/{location}", h.GetSiteLocations)
mux.HandleFunc("GET /site/{location}/", h.GetSiteLocations)

mux.HandleFunc("GET /mark", h.GetMarks)
mux.HandleFunc("GET /mark/", h.GetMarks)
mux.HandleFunc("GET /mark/{code}", h.GetMarkCode)
mux.HandleFunc("GET /mark/{code}/", h.GetMarkCode)

mux.HandleFunc("GET /monument", h.GetMonuments)
mux.HandleFunc("GET /monument/", h.GetMonuments)
mux.HandleFunc("GET /monument/{mark}", h.GetMonumentMark)
mux.HandleFunc("GET /monument/{mark}/", h.GetMonumentMark)

mux.HandleFunc("GET /sample", h.GetSamples)
mux.HandleFunc("GET /sample/", h.GetSamples)
mux.HandleFunc("GET /sample/{code}", h.GetSampleCode)
mux.HandleFunc("GET /sample/{code}/", h.GetSampleCode)
mux.HandleFunc("GET /sample/{code}/point", h.GetSamplePoints)
mux.HandleFunc("GET /sample/{code}/point/", h.GetSamplePoints)
mux.HandleFunc("GET /sample/{code}/point/{location}", h.GetSamplePointLocation)
mux.HandleFunc("GET /sample/{code}/point/{location}/", h.GetSamplePointLocation)
mux.HandleFunc("GET /sample/{code}/point/{location}/sensor", h.GetSamplePointLocationSensors)
mux.HandleFunc("GET /sample/{code}/point/{location}/sensor/", h.GetSamplePointLocationSensors)

mux.HandleFunc("GET /sensor", h.GetSensors)
mux.HandleFunc("GET /sensor/", h.GetSensors)
Expand Down
2 changes: 1 addition & 1 deletion cmd/deltadb/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ func main() {
if settings.db == "" || settings.init {

// insert extra response files
extra := set.KeyValue(settings.response, "Name", "Response", values)
extra := set.KeyValue(settings.response, "Response", "XML", values)

log.Println("initialise database")
start := time.Now()
Expand Down
47 changes: 34 additions & 13 deletions cmd/deltadb/mark.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,53 @@ import (
"encoding/json"
"log"
"net/http"

"github.com/GeoNet/delta/meta/sqlite"
"time"
)

type Mark struct {
Mark string `json:"station"`
Network string `json:"network"`
Igs string `json:"igs"`
Name string `json:"name"`
Latitude float64 `json:"latitude"`
Longitude float64 `json:"longitude"`
Elevation *float64 `json:"elevation,omitempty"`
Datum string `json:"datum"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
}

func (h handler) GetMarks(w http.ResponseWriter, r *http.Request) {

marks, err := sqlite.Marks(r.Context(), h.db)
query := "SELECT mark.mark,network.network,mark.igs,mark.name,mark.latitude,mark.longitude,mark.elevation,datum.datum,mark.start_date,mark.end_date FROM mark, datum, network, mark_network WHERE mark.datum_id = datum.datum_id AND mark.mark_id = mark_network.mark_id AND network.network_id = mark_network.network_id"
stmt, err := h.db.PrepareContext(r.Context(), query)
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer stmt.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(marks); err != nil {
log.Println(err)
results, err := stmt.QueryContext(r.Context())
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
}
defer results.Close()

func (h handler) GetMarkCode(w http.ResponseWriter, r *http.Request) {

code := r.PathValue("code")
marks := make([]Mark, 0)
for results.Next() {
var mark Mark
if err := results.Scan(&mark.Mark, &mark.Network, &mark.Igs, &mark.Name, &mark.Latitude, &mark.Longitude, &mark.Elevation, &mark.Datum, &mark.Start, &mark.End); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
marks = append(marks, mark)
}

marks, err := sqlite.Marks(r.Context(), h.db, sqlite.Code(code))
if err != nil {
if err = results.Err(); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
Expand Down
48 changes: 35 additions & 13 deletions cmd/deltadb/monument.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,32 +4,54 @@ import (
"encoding/json"
"log"
"net/http"

"github.com/GeoNet/delta/meta/sqlite"
"time"
)

type Monument struct {
Mark string `json:"mark"`
DomesNumber string `json:"domes_number,omitempty"`
MarkType string `json:"mark_type"`
Type string `json:"type"`
GroundRelationship float64 `json:"ground_relationship"`
FoundationType string `json:"foundation_type"`
FoundationDepth float64 `json:"foundation_depth"`
Bedrock string `json:"bedrock,omitempty"`
Geology string `json:"geology,omitempty"`
Start time.Time `json:"start"`
End time.Time `json:"end"`
}

func (h handler) GetMonuments(w http.ResponseWriter, r *http.Request) {

monuments, err := sqlite.Monuments(r.Context(), h.db)
query := "SELECT mark.mark,monument.domes_number,mark_type.mark_type,monument_type.monument_type,monument.ground_relationship,foundation_type.foundation_type,monument.foundation_depth,bedrock.bedrock,geology.geology,monument.start_date,monument.end_date FROM mark, monument, mark_type, monument_type, foundation_type, bedrock, geology WHERE mark.mark_id = monument.mark_id AND monument.mark_type_id = mark_type.mark_type_id AND monument.monument_type_id = monument_type.monument_type_id AND monument.foundation_type_id = foundation_type.foundation_type_id AND monument.bedrock_id = bedrock.bedrock_id AND monument.geology_id = geology.geology_id"
stmt, err := h.db.PrepareContext(r.Context(), query)
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
defer stmt.Close()

w.Header().Add("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
if err := json.NewEncoder(w).Encode(monuments); err != nil {
log.Println(err)
results, err := stmt.QueryContext(r.Context())
if err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
}
defer results.Close()

func (h handler) GetMonumentMark(w http.ResponseWriter, r *http.Request) {

mark := r.PathValue("mark")
monuments := make([]Monument, 0)
for results.Next() {
var monument Monument
if err := results.Scan(&monument.Mark, &monument.DomesNumber, &monument.MarkType, &monument.Type, &monument.GroundRelationship, &monument.FoundationType, &monument.FoundationDepth, &monument.Bedrock, &monument.Geology, &monument.Start, &monument.End); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
}
monuments = append(monuments, monument)
}

monuments, err := sqlite.Monuments(r.Context(), h.db, sqlite.Mark(mark))
if err != nil {
if err = results.Err(); err != nil {
log.Println("failed to execute query", err)
w.WriteHeader(500)
return
Expand Down
Loading

0 comments on commit 5615c7a

Please sign in to comment.