Test STL containers. (#31)

* Test STL containers.

* Start working on interpreting STL containers in TTrees.

* Added 'context' to 'basket_array'.

* All of the STL collection branches have the right interpretations.

* 'context' tricks to read headers outside of TTrees but not inside.

* Reading nested vectors, too. Got rid of the 'multiplicity' thing.

* AsSTLContainers are explicitly labled by whether they read headers or not.

* Deserialized first std::map in TTree.

* All STL collections can now be read.

* SAVE WORK; everything will break.

* fParentName-based solution to finding streamers.

* Remove commented-out bad code and fix filename.

* Interpretations produce typenames now.

* Another string case works.

* All of the STL tests work; only 3 (non-STL) are skip/FIXME.

tests/test_0018-array-fetching-interface.py

@@ -56,37 +56,37 @@ def test_leaf_interpretation():
 
         assert sample["n"].typename == "int32_t"
         assert sample["b"].typename == "bool"
-        assert sample["ab"].typename == "bool[]"
+        assert sample["ab"].typename == "bool[3]"
         assert sample["Ab"].typename == "bool[]"
         assert sample["i1"].typename == "int8_t"
-        assert sample["ai1"].typename == "int8_t[]"
+        assert sample["ai1"].typename == "int8_t[3]"
         assert sample["Ai1"].typename == "int8_t[]"
         assert sample["u1"].typename == "uint8_t"
-        assert sample["au1"].typename == "uint8_t[]"
+        assert sample["au1"].typename == "uint8_t[3]"
         assert sample["Au1"].typename == "uint8_t[]"
         assert sample["i2"].typename == "int16_t"
-        assert sample["ai2"].typename == "int16_t[]"
+        assert sample["ai2"].typename == "int16_t[3]"
         assert sample["Ai2"].typename == "int16_t[]"
         assert sample["u2"].typename == "uint16_t"
-        assert sample["au2"].typename == "uint16_t[]"
+        assert sample["au2"].typename == "uint16_t[3]"
         assert sample["Au2"].typename == "uint16_t[]"
         assert sample["i4"].typename == "int32_t"
-        assert sample["ai4"].typename == "int32_t[]"
+        assert sample["ai4"].typename == "int32_t[3]"
         assert sample["Ai4"].typename == "int32_t[]"
         assert sample["u4"].typename == "uint32_t"
-        assert sample["au4"].typename == "uint32_t[]"
+        assert sample["au4"].typename == "uint32_t[3]"
         assert sample["Au4"].typename == "uint32_t[]"
         assert sample["i8"].typename == "int64_t"
-        assert sample["ai8"].typename == "int64_t[]"
+        assert sample["ai8"].typename == "int64_t[3]"
         assert sample["Ai8"].typename == "int64_t[]"
         assert sample["u8"].typename == "uint64_t"
-        assert sample["au8"].typename == "uint64_t[]"
+        assert sample["au8"].typename == "uint64_t[3]"
         assert sample["Au8"].typename == "uint64_t[]"
         assert sample["f4"].typename == "float"
-        assert sample["af4"].typename == "float[]"
+        assert sample["af4"].typename == "float[3]"
         assert sample["Af4"].typename == "float[]"
         assert sample["f8"].typename == "double"
-        assert sample["af8"].typename == "double[]"
+        assert sample["af8"].typename == "double[3]"
         assert sample["Af8"].typename == "double[]"
 
 

tests/test_0023-more-interpretations-1.py

@@ -49,7 +49,6 @@ def test_strings1():
         assert result.tolist() == ["hey-{0}".format(i) for i in range(30)]
 
 
-@pytest.mark.skip(reason="FIXME: implement std::vector<std::string>")
 def test_strings4():
     with uproot4.open(
         skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root")
@@ -71,7 +70,6 @@ def test_strings4():
         ]
 
 
-@pytest.mark.skip(reason="FIXME: implement std::vector<std::vector<double>>")
 def test_strings4():
     with uproot4.open(skhep_testdata.data_path("uproot-vectorVectorDouble.root"))[
         "t/x"

tests/test_0028-fallback-to-read-streamer.py

@@ -10,11 +10,11 @@
 
 
 def test_fallback_reading():
-    # with uproot4.open(
-    #     skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root")
-    # ) as f:
-    #     f["tree:evt/P3/P3.Py"]
-    #     assert f.file._streamers is None
+    with uproot4.open(
+        skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root")
+    ) as f:
+        f["tree:evt/P3/P3.Py"]
+        assert f.file._streamers is None
 
     with uproot4.open(skhep_testdata.data_path("uproot-demo-double32.root")) as f:
         f["T/fD64"]

tests/test_0029-more-string-types.py

@@ -10,7 +10,7 @@
 import skhep_testdata
 
 import uproot4
-from uproot4.stl_containers import parse_typename
+from uproot4.interpretation.identify import parse_typename
 from uproot4.stl_containers import AsString
 from uproot4.stl_containers import AsVector
 from uproot4.stl_containers import AsSet
@@ -19,45 +19,59 @@
 
 def test_parse_typename():
     assert parse_typename("TTree") is uproot4.classes["TTree"]
-    assert parse_typename("string") == AsString()
-    assert parse_typename("std::string") == AsString()
-    assert parse_typename("std :: string") == AsString()
-    assert parse_typename("char*") == AsString(is_stl=False)
-    assert parse_typename("char *") == AsString(is_stl=False)
-    assert parse_typename("TString") == AsString(is_stl=False)
-    assert parse_typename("vector<TTree>") == AsVector(uproot4.classes["TTree"])
-    assert parse_typename("vector<int>") == AsVector(">i4")
-    assert parse_typename("vector<bool>") == AsVector("?")
-    assert parse_typename("vector<string>") == AsVector(AsString())
-    assert parse_typename("vector  <   string   >") == AsVector(AsString())
-    assert parse_typename("std::vector<std::string>") == AsVector(AsString())
-    assert parse_typename("vector<vector<int>>") == AsVector(AsVector(">i4"))
-    assert parse_typename("vector<vector<string>>") == AsVector(AsVector(AsString()))
+    assert parse_typename("string") == AsString(False)
+    assert parse_typename("std::string") == AsString(False)
+    assert parse_typename("std :: string") == AsString(False)
+    assert parse_typename("char*") == AsString(False)
+    assert parse_typename("char *") == AsString(False)
+    assert parse_typename("TString") == AsString(False)
+    assert parse_typename("vector<TTree>") == AsVector(True, uproot4.classes["TTree"])
+    assert parse_typename("vector<int>") == AsVector(True, ">i4")
+    assert parse_typename("vector<bool>") == AsVector(True, "?")
+    assert parse_typename("vector<string>") == AsVector(True, AsString(False))
+    assert parse_typename("vector  <   string   >") == AsVector(True, AsString(False))
+    assert parse_typename("std::vector<std::string>") == AsVector(True, AsString(False))
+    assert parse_typename("vector<vector<int>>") == AsVector(
+        True, AsVector(False, ">i4")
+    )
+    assert parse_typename("vector<vector<string>>") == AsVector(
+        True, AsVector(False, AsString(False))
+    )
     assert parse_typename("vector<vector<char*>>") == AsVector(
-        AsVector(AsString(is_stl=False))
+        True, AsVector(False, AsString(False))
+    )
+    assert parse_typename("set<unsigned short>") == AsSet(True, ">u2")
+    assert parse_typename("std::set<unsigned short>") == AsSet(True, ">u2")
+    assert parse_typename("set<string>") == AsSet(True, AsString(False))
+    assert parse_typename("set<vector<string>>") == AsSet(
+        True, AsVector(False, AsString(False))
+    )
+    assert parse_typename("set<vector<string> >") == AsSet(
+        True, AsVector(False, AsString(False))
+    )
+    assert parse_typename("map<int, double>") == AsMap(True, ">i4", ">f8")
+    assert parse_typename("map<string, double>") == AsMap(True, AsString(True), ">f8")
+    assert parse_typename("map<int, string>") == AsMap(True, ">i4", AsString(True))
+    assert parse_typename("map<string, string>") == AsMap(
+        True, AsString(True), AsString(True)
+    )
+    assert parse_typename("map<string,string>") == AsMap(
+        True, AsString(True), AsString(True)
+    )
+    assert parse_typename("map<   string,string   >") == AsMap(
+        True, AsString(True), AsString(True)
     )
-    assert parse_typename("set<unsigned short>") == AsSet(">u2")
-    assert parse_typename("std::set<unsigned short>") == AsSet(">u2")
-    assert parse_typename("set<string>") == AsSet(AsString())
-    assert parse_typename("set<vector<string>>") == AsSet(AsVector(AsString()))
-    assert parse_typename("set<vector<string> >") == AsSet(AsVector(AsString()))
-    assert parse_typename("map<int, double>") == AsMap(">i4", ">f8")
-    assert parse_typename("map<string, double>") == AsMap(AsString(), ">f8")
-    assert parse_typename("map<int, string>") == AsMap(">i4", AsString())
-    assert parse_typename("map<string, string>") == AsMap(AsString(), AsString())
-    assert parse_typename("map<string,string>") == AsMap(AsString(), AsString())
-    assert parse_typename("map<   string,string   >") == AsMap(AsString(), AsString())
     assert parse_typename("map<string,vector<int>>") == AsMap(
-        AsString(), AsVector(">i4")
+        True, AsString(True), AsVector(True, ">i4")
     )
     assert parse_typename("map<vector<int>, string>") == AsMap(
-        AsVector(">i4"), AsString()
+        True, AsVector(True, ">i4"), AsString(True)
     )
     assert parse_typename("map<vector<int>, set<float>>") == AsMap(
-        AsVector(">i4"), AsSet(">f4")
+        True, AsVector(True, ">i4"), AsSet(True, ">f4")
     )
     assert parse_typename("map<vector<int>, set<set<float>>>") == AsMap(
-        AsVector(">i4"), AsSet(AsSet(">f4"))
+        True, AsVector(True, ">i4"), AsSet(True, AsSet(False, ">f4"))
     )
 
     with pytest.raises(ValueError):
@@ -125,37 +139,46 @@ def test_map_string_string_in_object():
         }
 
 
-@pytest.mark.skip(
-    reason="FIXME: test works, but the file is not in scikit-hep-testdata yet"
-)
 def test_map_long_int_in_object():
-    with uproot4.open(
-        "/home/pivarski/irishep/scikit-hep-testdata/src/skhep_testdata/data/uproot-issue283.root"
-    ) as f:
-        print(f["config/detector"])
-
-    # raise Exception
-
+    with uproot4.open(skhep_testdata.data_path("uproot-issue283.root")) as f:
+        map_long_int = f["config/detector"].member("ChannelIDMap")
+        assert (map_long_int.keys().min(), map_long_int.keys().max()) == (
+            46612627560,
+            281410180683757,
+        )
+        assert (map_long_int.values().min(), map_long_int.values().max()) == (0, 5159)
+
+
+def test_top_level_vectors():
+    with uproot4.open(skhep_testdata.data_path("uproot-issue38a.root"))[
+        "ntupler/tree"
+    ] as tree:
+        assert [x.tolist() for x in tree["v_int16"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_int16"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_int32"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_int64"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_uint16"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_uint32"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_uint64"].array(library="np")] == [[1, 2, 3]]
+        assert [x.tolist() for x in tree["v_bool"].array(library="np")] == [
+            [False, True]
+        ]
+        assert [x.tolist() for x in tree["v_float"].array(library="np")] == [
+            [999.0, -999.0]
+        ]
+        assert [x.tolist() for x in tree["v_double"].array(library="np")] == [
+            [999.0, -999.0]
+        ]
 
-# has STL vectors at top-level:
-#
-# python -c 'import uproot; t = uproot.open("/home/pivarski/irishep/scikit-hep-testdata/src/skhep_testdata/data/uproot-issue38a.root")["ntupler/tree"]; print("\n".join(str((x._fName, getattr(x, "_fStreamerType", None), getattr(x, "_fClassName", None), getattr(x, "_fType", None), x.interpretation)) for x in t.allvalues()))'
 
-# has STL map<int,struct> as described here:
-#
-# https://github.com/scikit-hep/uproot/issues/468#issuecomment-646325842
-#
-# python -c 'import uproot; t = uproot.open("/home/pivarski/irishep/scikit-hep-testdata/src/skhep_testdata/data/uproot-issue468.root")["Geant4Data/Geant4Data./Geant4Data.particles"]; print(t.array(uproot.asdebug)[0][:1000])'
-
-# def test_strings1():
-#     with uproot4.open(
-#         skhep_testdata.data_path("uproot-issue31.root")
-#     )["T/name"] as branch:
-#         result = branch.array(library="np")
-#         assert result.tolist() == ["one", "two", "three", "four", "five"]
+def test_strings1():
+    with uproot4.open(skhep_testdata.data_path("uproot-issue31.root"))[
+        "T/name"
+    ] as branch:
+        result = branch.array(library="np")
+        assert result.tolist() == ["one", "two", "three", "four", "five"]
 
 
-@pytest.mark.skip(reason="FIXME: implement strings specified by a TStreamer")
 def test_strings2():
     with uproot4.open(
         skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root")
@@ -164,7 +187,6 @@ def test_strings2():
         assert result.tolist() == ["evt-{0:03d}".format(i) for i in range(100)]
 
 
-@pytest.mark.skip(reason="FIXME: implement std::string")
 def test_strings3():
     with uproot4.open(
         skhep_testdata.data_path("uproot-small-evnt-tree-fullsplit.root")