Refactor Cliopatria loading and visualiser notebook

cliopatria/convert_data.py

@@ -0,0 +1,64 @@
+import geopandas as gpd
+from distinctipy import get_colors, get_hex
+import sys
+
+def cliopatria_gdf(gdf):
+    """
+    Load the Cliopatria shape dataset with GeoPandas, process names and colors efficiently.
+    """
+
+    # Generate DisplayName for each shape based on the 'Name' field
+    gdf['DisplayName'] = gdf['Name'].str.replace('[()]', '', regex=True)
+
+    # Add type prefix to DisplayName where type is not 'POLITY'
+    gdf.loc[gdf['Type'] != 'POLITY', 'DisplayName'] = gdf['Type'].str.capitalize() + ': ' + gdf['DisplayName']
+
+    print(f"Generated shape names for {len(gdf)} shapes.")
+    print("Assigning colours to shapes...")
+
+    # Use DistinctiPy package to assign a colour based on the DisplayName field
+    colour_keys = gdf['DisplayName'].unique()
+    colours = [get_hex(col) for col in get_colors(len(colour_keys))]
+    colour_mapping = dict(zip(colour_keys, colours))
+
+    # Map colors to a new column
+    gdf['Color'] = gdf['DisplayName'].map(colour_mapping)
+
+    print(f"Assigned colours to {len(gdf)} shapes.")
+    print("Determining polity start and end years...")
+
+    # Add a column called 'PolityStartYear' to the GeoDataFrame which is the minimum 'FromYear' of all shapes with the same 'Name'
+    gdf['PolityStartYear'] = gdf.groupby('Name')['FromYear'].transform('min')
+
+    # Add a column called 'PolityEndYear' to the GeoDataFrame which is the maximum 'ToYear' of all shapes with the same 'Name'
+    gdf['PolityEndYear'] = gdf.groupby('Name')['ToYear'].transform('max')
+
+    print(f"Determined polity start and end years for {len(gdf)} shapes.")
+
+    return gdf
+
+
+# Check if a GeoJSON file path was provided as a command line argument
+if len(sys.argv) < 2:
+    print("Please provide the path to the GeoJSON file as a command line argument.")
+    sys.exit(1)
+
+geojson_path = sys.argv[1]
+
+try:
+    gdf = gpd.read_file(geojson_path)
+except Exception as e:
+    print(f"Error loading GeoJSON file: {str(e)}")
+    sys.exit(1)
+
+# Call the cliopatria_gdf function to process the GeoDataFrame
+processed_gdf = cliopatria_gdf(gdf)
+
+# Save the processed GeoDataFrame as a new GeoJSON file
+output_path = geojson_path.replace('.geojson', '_seshat_processed.geojson')
+try:
+    processed_gdf.to_file(output_path, driver='GeoJSON')
+    print(f"Processed GeoDataFrame saved to: {output_path}")
+except Exception as e:
+    print(f"Error saving processed GeoDataFrame: {str(e)}")
+    sys.exit(1)

docs/source/getting-started/setup/spatialdb.rst

@@ -25,14 +25,20 @@ Cliopatria shape dataset
 -------------------------
 
 ..
-    TODO: Add a link here to the published Clipatria dataset
+    TODO: Add a link here to the published Cliopatria dataset
 
 1. Download and unzip the Cliopatria dataset.
-2. Populate ``core_videoshapefile`` table using the following command:
+2. Update the Cliopatria GeoJSON file with colours and other properties required by Seshat:
+
+      .. code-block:: bash
+
+         $ python cliopatria/convert_data.py /path/to/cliopatria.geojson
+   Note: this will create a new file with the same name but with the suffix "_seshat_processed.geojson"
+3. Populate ``core_videoshapefile`` table using the following command:
 
    .. code-block:: bash
 
-      $ python manage.py populate_videodata /path/to/data
+      $ python manage.py populate_videodata /path/to/cliopatria_seshat_processed.geojson
 
    Note: if you wish to further simplify the Cliopatria shape resolution used by the world map after loading it into the database, open ``seshat/apps/core/management/commands/populate_videodata.py`` and modify the SQL query under the comment: "Adjust the tolerance param of ST_Simplify as needed"