Add commodities

commodities/config.yml

@@ -0,0 +1,51 @@
+- name: wheat
+  chart_id: 2534
+  multiply_by: 0.036 # bushel to kg (https://www.agric.gov.ab.ca/app19/calc/crop/bushel2tonne.jsp)
+- name: corn
+  chart_id: 2532
+  multiply_by: 0.039 # bushel to kg (https://www.agric.gov.ab.ca/app19/calc/crop/bushel2tonne.jsp)
+- name: soybean
+  chart_id: 2531
+  multiply_by: 0.036 # bushel to kg (https://www.agric.gov.ab.ca/app19/calc/crop/bushel2tonne.jsp)
+- name: oats
+  chart_id: 2536
+  multiply_by: 0.064 # bushel to kg (https://www.agric.gov.ab.ca/app19/calc/crop/bushel2tonne.jsp)
+- name: coffee
+  chart_id: 2535
+  multiply_by: 2.205 # pound to kg
+- name: cotton
+  chart_id: 2533
+  multiply_by: 2.205 # pound to kg
+- name: sugar
+  chart_id: 2537
+  multiply_by: 2.205 # pound to kg
+- name: soybean_oil
+  chart_id: 2538
+  multiply_by: 2.205 # pound to kg
+- name: heating_oil
+  chart_id: 2479
+  multiply_by: 0.264172 # gallon to liter
+- name: natural_gas
+  chart_id: 2478
+  multiply_by: 1
+- name: brent_crude_oil
+  chart_id: 2480
+  multiply_by: 1
+- name: long_term_crude_oil
+  chart_id: 1369
+  multiply_by: 1
+- name: copper
+  chart_id: 1476
+  multiply_by: 2.205 # pound to kg
+- name: gold
+  chart_id: 1333
+  multiply_by: 0.035274 # ounce to g
+- name: silver
+  chart_id: 1470
+  multiply_by: 0.035274 # ounce to g
+- name: platinum
+  chart_id: 2540
+  multiply_by: 0.035274 # ounce to g
+- name: palladium
+  chart_id: 2542
+  multiply_by: 0.035274 # ounce to g

commodities/main.py

@@ -0,0 +1,58 @@
+from functools import reduce
+import json
+import re
+import requests
+import yaml
+
+from bs4 import BeautifulSoup
+import pandas as pd
+from tqdm import tqdm
+
+
+def get_config():
+    with open("config.yml") as file:
+        config = yaml.load(file, Loader=yaml.FullLoader)
+    return config
+
+
+def get_original_data(chart_id: int) -> pd.DataFrame:
+    url = f"https://www.macrotrends.net/assets/php/chart_iframe_comp.php?id={chart_id}"
+    soup = BeautifulSoup(requests.get(url).content, "html.parser")
+    for script in soup.find_all("script"):
+        if script.string:
+            if "originalData" in script.string:
+                data = re.search(r"originalData = ([^;]+)", script.string).group(1)
+                break
+    return pd.DataFrame.from_records(json.loads(data))
+
+
+def clean_data(df: pd.DataFrame, name: str, multiply_by: float) -> pd.DataFrame:
+    if "close1" in df.columns:
+        df = df.drop(columns="close").rename(columns={"close1": "close"})
+    df["close"] = df.close.astype(float).mul(multiply_by).round(3)
+    return df.dropna(subset="close").drop(columns="id").rename(columns={"close": name})
+
+
+def main():
+    commodities = get_config()
+
+    dataframes = []
+    for com in tqdm(commodities):
+        df = get_original_data(com["chart_id"]).pipe(
+            clean_data, com["name"], com["multiply_by"]
+        )
+        dataframes.append(df)
+
+    df = reduce(
+        lambda df1, df2: pd.merge(
+            df1, df2, on="date", how="outer", validate="one_to_one"
+        ),
+        dataframes,
+    ).sort_values("date")
+    df.insert(0, "entity", "World")
+
+    df.to_csv("output/commodities.csv", index=False)
+
+
+if __name__ == "__main__":
+    main()