cube_triple_generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Import toolboxes, mapping files and RDF prefices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\"\"\"\n",
    "Created on Mon Jul 02 10:42:14 2018\n",
    "\n",
    "@author: rthomas\n",
    "\n",
    "V6 updates:\n",
    "    Added qb:codeList for eg:station_id and eg:statistic.\n",
    "    Added qb:CodedProperty as type for eg:station_id and eg:statistic\n",
    "    Changed rdfs:seeAlso to qb:concept for P07 terms linked to qb:MeasuredProperty(s)\n",
    "\"\"\"\n",
    "import datetime\n",
    "import os\n",
    "import pandas as pd\n",
    "      \n",
    "# Set working directory\n",
    "working = 'https://raw.githubusercontent.com/IrishMarineInstitute/OGI_MIPilot/master/'\n",
    "\n",
    "# Load vocab mappings for measured properties\n",
    "mapfile = working+'mappings/measures.csv'\n",
    "mapping = pd.read_csv(mapfile)\n",
    "\n",
    "# Load details of dimension properties\n",
    "stnfile = working+'mappings/dimension_station_id.csv'\n",
    "stations = pd.read_csv(stnfile)\n",
    "\n",
    "statfile = working+'mappings/dimension_statistic.csv'\n",
    "statistics = pd.read_csv(statfile)\n",
    "\n",
    "# Load details of datasets to be QB'd\n",
    "dsetfile = working+'mappings/dsets2QB.csv'\n",
    "datasets = pd.read_csv(dsetfile)\n",
    "\n",
    "dset_ids = datasets.dset_id.tolist()\n",
    "    \n",
    "#%% Set global variables\n",
    "now = datetime.date.today()\n",
    "\n",
    "# Set output filepath\n",
    "ttl_vocabs = os.path.join(working,'rdf/MI_QB_vocabs.ttl')\n",
    "\n",
    "# Construct RDF Data Cube\n",
    "prefices = \"\"\"@prefix rdf:      <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .\n",
    "@prefix rdfs:     <http://www.w3.org/2000/01/rdf-schema#> .\n",
    "@prefix owl:      <http://www.w3.org/2002/07/owl#> .\n",
    "@prefix xsd:      <http://www.w3.org/2001/XMLSchema#> .\n",
    "@prefix skos:     <http://www.w3.org/2004/02/skos/core#> .\n",
    "@prefix void:     <http://rdfs.org/ns/void#> .\n",
    "@prefix dcat:     <http://www.w3.org/ns/dcat#> .\n",
    "@prefix dct:      <http://purl.org/dc/terms/> .\n",
    "@prefix foaf:     <http://xmlns.com/foaf/0.1/> .\n",
    "@prefix org:      <http://www.w3.org/ns/org#> .\n",
    "@prefix geo:      <http://www.opengis.net/ont/geosparql#> .\n",
    "@prefix prov:     <http://www.w3.org/ns/prov#> .\n",
    "\n",
    "@prefix qb:       <http://purl.org/linked-data/cube#> .\n",
    "\n",
    "@prefix sdmx-concept:    <http://purl.org/linked-data/sdmx/2009/concept#> .\n",
    "@prefix sdmx-dimension:  <http://purl.org/linked-data/sdmx/2009/dimension#> .\n",
    "@prefix sdmx-attribute:  <http://purl.org/linked-data/sdmx/2009/attribute#> .\n",
    "@prefix sdmx-measure:    <http://purl.org/linked-data/sdmx/2009/measure#> .\n",
    "@prefix sdmx-metadata:   <http://purl.org/linked-data/sdmx/2009/metadata#> .\n",
    "@prefix sdmx-code:       <http://purl.org/linked-data/sdmx/2009/code#> .\n",
    "@prefix sdmx-subject:    <http://purl.org/linked-data/sdmx/2009/subject#> .\n",
    "\n",
    "@prefix eg:              <http://data.marine.ie/datacube#> .\n",
    "@prefix mi-vcb:          <http://data.marine.ie/vocab#> .\n",
    "\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the SKOS collections to be used by the Data Cubes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define SKOS collections for the dataset\n",
    "skos_station = \"\"\"\n",
    "mi-vcb:Station_id a rdfs:Class, owl:Class ;\n",
    "\\trdfs:subClassOf skos:Concept ;\n",
    "\\trdfs:label \"Code list for Station codelist class\"@en ;\n",
    "\\trdfs:comment \"This code list provides the station details.\"@en ;\n",
    "\\trdfs:seeAlso mi-vcb:station_id .\n",
    "    \n",
    "mi-vcb:station_id a skos:ConceptScheme;\n",
    "\\tskos:prefLabel \"Code list for stations in the data cube - codelist scheme\"@en ;\n",
    "\\trdfs:label \"Code list for stations in the data cube - codelist scheme\"@en ;\n",
    "\\tskos:notation \"Summary statistic\"@en ;\n",
    "\\tskos:note \"This code list provides the attributes for the stations in the data cube.\"@en ;\n",
    "\\tskos:definition \"This code list provides the attributes for the stations in the data cube.\"@en ;\n",
    "\\trdfs:seeAlso mi-vcb:Station_id ;\n",
    "\"\"\"\n",
    "    \n",
    "for i in range(0,len(stations)):\n",
    "    text = \"\\tskos:member mi-vcb:%s ;\\n\" % (stations.loc[i]['Label'])\n",
    "    skos_station = skos_station + text\n",
    "\n",
    "skos_station = skos_station + \"\\t.\\n\"\n",
    "\n",
    "for i in range(0,len(stations)):\n",
    "    label = stations.loc[i]['Label']\n",
    "    name = stations.loc[i]['Name']\n",
    "    definition = stations.loc[i]['Definition']\n",
    "    lat = stations.loc[i]['Lat']\n",
    "    lon = stations.loc[i]['Lon']\n",
    "    text = \"\"\"mi-vcb:%s a skos:Concept, mi-vcb:Station_id, geo:Feature ;\n",
    "    \\tskos:topConceptOf mi-vcb:station_id ;\n",
    "    \\tskos:prefLabel \"%s\"@en ;\n",
    "    \\tskos:notation \"%s\"@en ;\n",
    "    \\tskos:definition \"Narative description of the site here\"@en ;\n",
    "    \\tskos:inScheme mi-vcb:station_id ;\n",
    "    \\tgeo:hasGeometry mi-vcb:%s-geom .\n",
    "mi-vcb:%s-geom geo:asWKT \"POINT(%s %s)\"^^geo:wktLiteral .\\n\\n\"\"\" % (label,name,name,label,label,lon,lat)\n",
    "    skos_station = skos_station + text\n",
    "\n",
    "skos_statistic = \"\"\"\n",
    "mi-vcb:Statistic a rdfs:Class, owl:Class ;\n",
    "\\trdfs:subClassOf skos:Concept ;\n",
    "\\trdfs:label \"Code list for Statistic codelist class\"@en ;\n",
    "\\trdfs:comment \"This  code list provides the summary statistic.\"@en ;\n",
    "\\trdfs:seeAlso mi-vcb:statistic .\n",
    "    \n",
    "mi-vcb:statistic a skos:ConceptScheme ;\n",
    "\\tskos:prefLabel \"Code list for statistic codelist scheme\"@en ;\n",
    "\\trdfs:label \"Code list for statistic codelist scheme\"@en ;\n",
    "\\tskos:notation \"Summary statistic\"@en ;\n",
    "\\tskos:note \"This code list provides the summary statistics provided in the dataset.\"@en ;\n",
    "\\tskos:definition <http://vocab.nerc.ac.uk/collection/S07/current/> ;\n",
    "\\trdfs:seeAlso mi-vcb:Statistic ;\\n\"\"\"\n",
    "    \n",
    "for i in range(0,len(statistics)):\n",
    "    text = \"\\tskos:member mi-vcb:%s ;\\n\" % (statistics.loc[i]['notation'])\n",
    "    skos_statistic = skos_statistic + text\n",
    "\n",
    "skos_statistic = skos_statistic + \"\\t.\\n\\n\"\n",
    "\n",
    "for i in range(0,len(statistics)):\n",
    "    notation = statistics.loc[i]['notation']\n",
    "    prefLabel = statistics.loc[i]['prefLabel']\n",
    "    definition = statistics.loc[i]['definition']\n",
    "    sameAs = statistics.loc[i]['sameAs']\n",
    "    if sameAs=='not applicable':\n",
    "        text = \"\"\"mi-vcb:%s a skos:Concept, mi-vcb:Statistic ;\n",
    "\\tskos:topConceptOf mi-vcb:statistic ;\n",
    "\\tskos:prefLabel \"%s\"@en ;\n",
    "\\tskos:notation \"%s\"@en ;\n",
    "\\tskos:definition \"%s\"@en ;\n",
    "\\tskos:inScheme mi-vcb:statistic ;\n",
    "\\t.\\n\\n\"\"\" % (notation,prefLabel,notation,definition)\n",
    "        skos_statistic = skos_statistic + text\n",
    "    else:\n",
    "        text = \"\"\"mi-vcb:%s a skos:Concept, mi-vcb:Statistic ;\n",
    "\\tskos:topConceptOf mi-vcb:statistic ;\n",
    "\\tskos:prefLabel \"%s\"@en ;\n",
    "\\tskos:notation \"%s\"@en ;\n",
    "\\tskos:definition \"%s\"@en ;\n",
    "\\tskos:sameAs <%s> ;\n",
    "\\tskos:inScheme mi-vcb:statistic ;\n",
    "\\t.\\n\\n\"\"\" % (notation,prefLabel,notation,definition,sameAs)\n",
    "        skos_statistic = skos_statistic + text\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(skos_station)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(skos_statistic)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the dimensions and measures of the Data Cubes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dimensions\n",
    "\n",
    "dimensions = \"\"\"# -- Dimensions and measures  ----------------------------\n",
    " \n",
    "eg:station_id  a rdf:Property, qb:DimensionProperty, qb:CodedProperty ;\n",
    "\\trdfs:label \"station_id\"@en ;\n",
    "\\trdfs:subPropertyOf sdmx-dimension:refArea ;\n",
    "\\tqb:codeList mi-vcb:station_id ;\n",
    "\\trdfs:range mi-vcb:station_id ;\n",
    "\\tqb:concept sdmx-concept:refArea ;\n",
    "\\t.\n",
    "\n",
    "eg:Date  a rdf:Property, qb:DimensionProperty ;\n",
    "\\trdfs:label \"Date\"@en ;\n",
    "\\trdfs:subPropertyOf sdmx-dimension:refPeriod ;\n",
    "\\trdfs:range xsd:date ;\n",
    "\\tqb:concept sdmx-concept:refPeriod ;\n",
    "\\t.\n",
    "    \n",
    "eg:statistic  a rdf:Property, qb:DimensionProperty, qb:CodedProperty ;\n",
    "\\trdfs:label \"statistic\"@en ;\n",
    "\\trdfs:subPropertyOf sdmx-dimension:statConcDef ;\n",
    "\\tqb:codeList mi-vcb:statistic ;\n",
    "\\trdfs:range mi-vcb:statistic ;\n",
    "\\tqb:concept sdmx-concept:statConcDef ;\n",
    "\\t.\n",
    "\"\"\"\n",
    "\n",
    "# Measures\n",
    "measures = \"\"\n",
    "for h in range(0,len(mapping)):\n",
    "    meas = str(mapping.iloc[h]['Local'])\n",
    "    P01 = str(mapping.iloc[h]['P01'])\n",
    "    P07 = str(mapping.iloc[h]['P07'])\n",
    "    P06 = str(mapping.iloc[h]['P06'])\n",
    "    comment = str(mapping.iloc[h]['Comment'])\n",
    "    \n",
    "    text = \"\"\"eg:%s  a rdf:Property, qb:MeasureProperty;\n",
    "    \\trdfs:label \"%s\"@en ;\\n\"\"\" % (meas, meas)\n",
    "    if P01 != 'nan':\n",
    "        text = text + \"\"\"\\trdfs:seeAlso <%s> ;\\n\"\"\" % (P01)\n",
    "    if P07 != 'nan':\n",
    "        text = text + \"\"\"\\tqb:concept <%s> ;\\n\"\"\" % (P07)\n",
    "    text = text + \"\"\"\\trdfs:comment \"%s\"@en ;\n",
    "    \\trdfs:subPropertyOf sdmx-measure:obsValue ;\n",
    "    \\tsdmx-attribute:unitMeasure <%s> ;\n",
    "    \\trdfs:range xsd:decimal ;\n",
    "    \\t.\\n\\n\"\"\" % (comment, P06)\n",
    "    measures = measures + text\n",
    "\n",
    "# Add unit labels and short names\n",
    "unit_text = \"\"\n",
    "    \n",
    "units = mapping[['P06', 'Unit_label', 'Unit_note']].reset_index(drop=True)\n",
    "units = units.drop_duplicates()\n",
    "\n",
    "for h in range(0,len(units)):\n",
    "    P06 = str(units.iloc[h]['P06'])\n",
    "    ulabel = str(units.iloc[h]['Unit_label'])\n",
    "    unote = str(units.iloc[h]['Unit_note'])\n",
    "    text = \"\"\"<%s> skos:prefLabel \"%s\"@en ;\n",
    "    \\tskos:altLabel \"%s\"@en ;\n",
    "    \\t.\n",
    "\"\"\" % (P06, ulabel, unote)\n",
    "    unit_text = unit_text + text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(dimensions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(measures)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(unit_text)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define the slices available for the Data Cube"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data Cube slices\n",
    "\n",
    "slice_keys = \"\"\"eg:sliceByStationID_statistic a qb:SliceKey;\n",
    "\\trdfs:label \"Slice by station-statistic combination\"@en;\n",
    "\\trdfs:comment \"Slice grouping by station and statistic\"@en;\n",
    "\\tqb:componentProperty eg:station_id, eg:statistic; \n",
    "\\t.\n",
    "\"\"\"\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Combine the vocabularies, dimension, measures and slice key triples into one file and save. Do not run from mybinder example as saving files is not possible."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "header = prefices + skos_station + skos_statistic + dimensions + measures + unit_text + slice_keys\n",
    "\n",
    "file = open(ttl_vocabs, \"w\")\n",
    "file.write(header)\n",
    "file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define mappings from vocabs for obervations in RDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stn_map = dict(zip(stations.Name.tolist(), ['mi-vcb:%s' % string for string in stations.Label.tolist()]))\n",
    "\n",
    "stat_map = dict(zip([string.replace('daily_','') for string in statistics.notation.tolist()], ['mi-vcb:%s' % string for string in statistics.notation.tolist()]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Run next 2 cells to see mappings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(stn_map)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(stat_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Next section works through building each cube.\n",
    "\n",
    "Example data input files are called from Github.\n",
    "\n",
    "Commands to save output to file have been commented out so the Notebook works with mybinder.org"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#%% Work through building each cube\n",
    "for typed in dset_ids:\n",
    "    # Load data from files\n",
    "    # Load daily data from file\n",
    "    dailyfile = working+'data/%s/%s_daily_summary.csv' % (typed,typed)\n",
    "    daily = pd.read_csv(dailyfile, index_col=[0, 1], header = [0, 1], parse_dates=True, infer_datetime_format=True)\n",
    "    # Load availability data from file\n",
    "    availfile = working+'data/%s/%s_data_availability.csv' % (typed,typed)\n",
    "    avail = pd.read_csv(availfile, index_col=[0, 1], header = [0, 1], parse_dates=True, infer_datetime_format=True)\n",
    "    \n",
    "    # Dictionary of data cubes to add to RDF\n",
    "    cube_list = {'daily': daily, 'avail': avail}\n",
    "    \n",
    "    for dset_type in cube_list.keys():\n",
    "        dset_data = cube_list[dset_type]\n",
    "        \n",
    "        short = datasets[datasets.dset_id==typed]['short'].values[0] + \"_%s\" % (dset_type)\n",
    "        edmed = datasets[datasets.dset_id==typed]['edmed'].values[0]\n",
    "        originator = datasets[datasets.dset_id==typed]['originator'].values[0]\n",
    "        publisher = datasets[datasets.dset_id==typed]['publisher'].values[0]\n",
    "        abstract = datasets[datasets.dset_id==typed]['abstract'].values[0]\n",
    "        if dset_type == 'avail':\n",
    "            title = datasets[datasets.dset_id==typed]['title'].values[0] + \" - Daily availability of data\"\n",
    "            description = \"\"\"Availability as a percentage by day of year for the %s measurements.\"\"\" % (datasets[datasets.dset_id==typed]['title'].values[0])\n",
    "            unitattach = \"DataSet\"\n",
    "            \n",
    "        elif dset_type == 'daily':\n",
    "            title = datasets[datasets.dset_id==typed]['title'].values[0] + \" - Daily Summary Statistics\"\n",
    "            description = \"\"\"Summary statistics (daily mean, standard deviation, minimun and maximum) by day of year for the %s measurements.\"\"\" % (datasets[datasets.dset_id==typed]['title'].values[0])\n",
    "            unitattach = \"MeasureProperty\"\n",
    "            \n",
    "    #%% Data structure definition\n",
    "            \n",
    "        dset_definition_pt1 = \"\"\"\\neg:dsd-%s a qb:DataStructureDefinition; \n",
    "    # The dimensions\n",
    "    \\tqb:component [ a qb:ComponentSpecification ; qb:componentProperty eg:station_id ; qb:dimension eg:station_id; qb:order 1; qb:componentAttachment qb:Slice ],\n",
    "    \\t             [ a qb:ComponentSpecification ; qb:componentProperty eg:Date ; qb:dimension eg:Date; qb:order 2 ],\n",
    "    \\t             [ a qb:ComponentSpecification ; qb:componentProperty eg:statistic ; qb:dimension eg:statistic; qb:order 3; qb:componentAttachment qb:Slice ];\n",
    "        \n",
    "    # The measure(s)\\n\"\"\" % (short)\n",
    "        for meas in dset_data.columns.levels[0].tolist():\n",
    "            line = \"\"\"\\tqb:component [ a qb:ComponentSpecification ; qb:componentProperty eg:%s ; qb:measure eg:%s];\\n\"\"\" % (meas, meas)\n",
    "            dset_definition_pt1 = dset_definition_pt1 + line\n",
    "        \n",
    "        dset_definition_pt2 =  \"\"\"  \n",
    "    # The attributes\n",
    "    \\tqb:component [ a qb:ComponentSpecification ; qb:componentProperty sdmx-attribute:unitMeasure ; qb:attribute sdmx-attribute:unitMeasure ; qb:componentAttachment qb:%s; ] ;\n",
    "\n",
    "    # The slices\n",
    "    \\tqb:sliceKey eg:sliceByStationID_statistic ;\n",
    "    .\n",
    "    \\n# Attribute and slice types\n",
    "    sdmx-attribute:unitMeasure a qb:AttributeProperty .    \n",
    "    eg:sliceByStationID_statistic a qb:SliceKey .    \n",
    "    \"\"\" % (unitattach)\n",
    "        \n",
    "        dset_definition = dset_definition_pt1 + dset_definition_pt2\n",
    "    \n",
    "    #%% Loop through slices and observations\n",
    "        cell_tot = len(dset_data.columns.levels[0].tolist()) * dset_data.shape[0]\n",
    "        cell_num = 0\n",
    "        idx = pd.IndexSlice\n",
    "        \n",
    "        slce_txt = \"\"\n",
    "        slce = 0\n",
    "        for stnid in dset_data.index.levels[0].tolist():\n",
    "            obs = \"\"\"\\n\"\"\"\n",
    "            for stat in dset_data.columns.levels[1].tolist():\n",
    "                data = dset_data.loc[stnid].loc[:, idx[:, stat]]\n",
    "                obs_head = \"\"\"\\n# Column %s %s\n",
    "eg:%s_slice%s a qb:Slice;\n",
    "\\tqb:sliceStructure  eg:sliceByStationID_statistic ;\n",
    "\\teg:station_id        %s ;\n",
    "\\teg:statistic         %s ;\n",
    "\\tqb:observation \"\"\" % (short, slce, short, slce, stn_map[stnid], stat_map[stat])\n",
    "            \n",
    "                for i in range(cell_num, cell_num+len(data)):\n",
    "                    obs_head = obs_head + \"\"\"eg:%s_o%s, \"\"\" % (short, str(i+1).zfill(len(str(cell_tot))))\n",
    "                    i = i + 1\n",
    "                obs_head = obs_head[0:-2] + \" ;    .\\n\" \n",
    "                obs = obs + obs_head\n",
    "                for i in range(0, len(data.index)):\n",
    "                    value = \"'%s'^^xsd:date\" % (data.index[i].strftime('%Y-%m-%d'))\n",
    "                    obs_granule = \"\"\"\\neg:%s_o%s a qb:Observation;\n",
    "\\tqb:dataSet          eg:%s ;\n",
    "\\teg:Date             %s ;\n",
    "\\teg:station_id       %s ;\n",
    "\\teg:statistic        %s ;\\n\"\"\" % (short, str(cell_num+1).zfill(len(str(cell_tot))), short, value, stn_map[stnid], stat_map[stat])\n",
    "    \n",
    "                    measures = data.columns.tolist()\n",
    "                    for meas in measures:\n",
    "                        value = data.iloc[i][meas]\n",
    "                        if str(value) == 'nan':\n",
    "                            value = -9999\n",
    "                        obs_line = \"\"\"\\teg:%s        %s ;\\n\"\"\" % (meas[0], value)\n",
    "                        obs_granule = obs_granule + obs_line    \n",
    "                    if unitattach == \"DataSet\":\n",
    "                        obs_granule = obs_granule + \"\"\"\\tsdmx-attribute:unitMeasure <http://vocab.nerc.ac.uk/collection/P06/current/UPCT/> ;\\n\\t.\\n\"\"\"\n",
    "                    else:\n",
    "                        obs_granule = obs_granule + \"\"\"\\t.\\n\"\"\"\n",
    "                    obs = obs + obs_granule\n",
    "                    cell_num = cell_num + 1\n",
    "                slce_txt = slce_txt + \"eg:%s_slice%s, \" % (short, slce)\n",
    "                slce = slce + 1\n",
    "            stn_file = prefices + obs\n",
    "            print('%s_%s_%s_datacube generated' % (typed, dset_type, stnid.replace(' ','_')))\n",
    "# Commented out to run with mybinder.org\n",
    "#            ttl_stn = os.path.join(working,'rdf/%s_%s_%s_datacube.ttl' % (typed, dset_type, stnid.replace(' ','_')))\n",
    "#            file = open(ttl_stn, \"w\")\n",
    "#            file.write(stn_file)\n",
    "#            file.close()\n",
    "        slce_txt = slce_txt[0:-2] + \" ;\"\n",
    "    \n",
    "    #%% Data Set description    \n",
    "        dset_desc = \"\"\"eg:%s a qb:DataSet, prov:Entity ;\n",
    "    dct:title       \"%s\"@en;\n",
    "    rdfs:label      \"%s\"@en;\n",
    "    rdfs:comment    \"%s\"@en;\n",
    "    dct:description \"%s\"@en;\n",
    "    dct:publisher   <http://edmo.seadatanet.org/%s>;\n",
    "    dct:issued      \"%s\"^^xsd:date;\n",
    "    prov:wasDerivedFrom <http://edmed.seadatanet.org/%s> ;\n",
    "    dct:subject\n",
    "    \\t<http://purl.org/linked-data/sdmx/2009/subject#3.1> ,\n",
    "    \\t<http://purl.org/linked-data/sdmx/2009/subject#2.4.2> ,\n",
    "    \\t<http://vocab.nerc.ac.uk/collection/C19/current/SVX00015/> ;\n",
    "    qb:structure eg:dsd-%s ;\n",
    "    qb:slice %s\n",
    "    .\n",
    "    \n",
    "<http://edmo.seadatanet.org/%s> a org:Organization, foaf:Agent;\n",
    "    rdfs:label \"Marine Institute\"@en .\n",
    "        \n",
    "<http://edmed.seadatanet.org/%s> a dcat:Dataset, prov:Entity . \"\"\" % (short, title, title, description, description, publisher, now.strftime('%Y-%m-%d'), edmed, short, slce_txt, publisher, edmed)\n",
    "    \n",
    "    #%% Combine all RDF sections for a data cube header    \n",
    "        dset_rdf = prefices + dset_desc + dset_definition\n",
    "        print('%s_%s_datacube_header generated' % (typed, dset_type))\n",
    "    \n",
    "    #%% Save RDF to file as ttl commented out fo use with mybinder.org\n",
    "#        ttl = os.path.join(working,'rdf/%s_%s_datacube_header.ttl' % (typed, dset_type))\n",
    "#        file = open(ttl, \"w\")\n",
    "#        file.write(dset_rdf)\n",
    "#        file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of the Data Cube description triples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(dset_rdf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of slice triples and link to observations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(obs[0:5000])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Example of observation triples."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(obs[200047:204687])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dset_data.index.levels[0].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}