Skip to content

Commit

Permalink
Update Makefile meson. (#574)
Browse files Browse the repository at this point in the history 
* Update Makefile meson.

* Fix tests.

* Change to release.

* Fix boostrap ci95.

* Fix boostrap rmse.

* Fix boostrap test.

* Fix boostrap test.

* Fix boostrap test; release.

* Update based on 1480c1 suggestion.
  • Loading branch information
@li-zhi
li-zhi authored May 15, 2020
1 parent 36b35b3 commit fe36d71
Show file tree
Hide file tree
Showing 3 changed files with 63 additions and 64 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ __pycache__/
build/
dist/
vmaf_output.xml
compile_commands.json
10 changes: 4 additions & 6 deletions Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
all:
cd third_party/libsvm && make lib

mkdir -p libvmaf/build && cd libvmaf/build && \
meson .. --buildtype release && \
ninja -vC .
meson setup libvmaf/build libvmaf --buildtype release && \
ninja -vC libvmaf/build

clean:
cd third_party/libsvm && make clean && cd -
rm -rf libvmaf/build

install:
mkdir -p libvmaf/build && cd libvmaf/build && \
meson .. --buildtype release && \
ninja -vC . install
meson setup libvmaf/build libvmaf --buildtype release && \
ninja -vC libvmaf/build install
116 changes: 58 additions & 58 deletions python/test/bootstrap_train_test_model_test.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,18 +51,18 @@ def test_train_predict_bootstrap_libsvmnusvr(self):
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.6226308662005923, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.6696474832672723, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.39703824367678103)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.3984211902006627)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825468723706)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.39703824367678103, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.3984211902006627, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825468723706, places=2)

model = BootstrapLibsvmNusvrTrainTestModel(
{'norm_type': 'clip_0to1'}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.8424314108711469, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.8035115098260148, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.34509843867201334)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.5003674768667943)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.742621319805588)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.34509843867201334, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.5003674768667943, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.742621319805588, places=2)

model = BootstrapLibsvmNusvrTrainTestModel(
{'norm_type': 'custom_clip_0to1',
Expand All @@ -73,27 +73,27 @@ def test_train_predict_bootstrap_libsvmnusvr(self):
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.8463916276654698, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.8056949768422423, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3510958063355727)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.479040231431352)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.758095707489271)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3510958063355727, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.479040231431352, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.758095707489271, places=2)

model = BootstrapLibsvmNusvrTrainTestModel(
{'norm_type': 'clip_minus1to1'}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.831435275234099, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.7833205217865182, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.37890490520445996)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.395027628418071)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.750492595014303)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.37890490520445996, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.395027628418071, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.750492595014303, places=2)

model = BootstrapLibsvmNusvrTrainTestModel(
{'norm_type': 'none'}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.23294283650716543, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.17423420271622292, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3505327957559016)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.6104498263465024)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.617611259682587)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3505327957559016, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.6104498263465024, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.617611259682587, places=2)

def test_train_save_load_predict_bootstrap_libsvmnusvr(self):

Expand All @@ -116,15 +116,15 @@ def test_train_save_load_predict_bootstrap_libsvmnusvr(self):

self.assertAlmostEqual(self.model.evaluate(xs, ys)['RMSE'], 0.6226308662005923, places=4)
self.assertAlmostEqual(self.model.evaluate_bagging(xs, ys)['RMSE'], 0.6696474832672723, places=4)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.39703824367678103)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.3984211902006627)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825468723706)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.39703824367678103, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.3984211902006627, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825468723706, places=2)

self.assertAlmostEqual(loaded_model.evaluate(xs, ys)['RMSE'], 0.6226313987163097, places=4)
self.assertAlmostEqual(loaded_model.evaluate_bagging(xs, ys)['RMSE'], 0.6696478863129723, places=4)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.3970382109813205)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.398421319902801)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825575324514)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.3970382109813205, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.398421319902801, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.753825575324514, places=2)

def test_train_across_model_stats_bootstraplibsvmnusvr(self):

Expand Down Expand Up @@ -173,20 +173,20 @@ def test_train_predict_bootstrap_randomforest(self):
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.17634739353518517, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.1734023350765026, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986, places=3)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333, places=1)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985, places=1)

model = BootstrapSklearnRandomForestTrainTestModel(
{'norm_type': 'clip_0to1',
'n_estimators': 10,
'random_state': 0}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.15634392282439, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16538552470948875, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3539523612820203)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4091111111111108)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.667805555555555)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16538552470948875, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3539523612820203, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4091111111111108, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.667805555555555, places=1)

model = BootstrapSklearnRandomForestTrainTestModel(
{'norm_type': 'custom_clip_0to1',
Expand All @@ -198,32 +198,32 @@ def test_train_predict_bootstrap_randomforest(self):
}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.15634392282439, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16538552470948875, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3539523612820203)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4091111111111108)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.667805555555555)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16538552470948875, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3539523612820203, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.4091111111111108, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.667805555555555, places=1)

model = BootstrapSklearnRandomForestTrainTestModel(
{'norm_type': 'clip_minus1to1',
'n_estimators': 10,
'random_state': 0}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.15634392282438947, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16634325875304712, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.35470969070245834)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.405222222222222)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.686916666666666)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16634325875304712, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.35470969070245834, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.405222222222222, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.686916666666666, places=2)

model = BootstrapSklearnRandomForestTrainTestModel(
{'norm_type': 'none',
'n_estimators': 10,
'random_state': 0}, None)
model.train(xys)
self.assertAlmostEqual(model.evaluate(xs, ys)['RMSE'], 0.15634392282438941, places=4)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16935240638616547, places=4)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3538762838023098)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.402638888888889)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.679027777777778)
self.assertAlmostEqual(model.evaluate_bagging(xs, ys)['RMSE'], 0.16935240638616547, places=3)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_stddev'], 0.3538762838023098, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_low'], 3.402638888888889, places=2)
self.assertAlmostEqual(model.evaluate_stddev(xs)['mean_ci95_high'], 4.679027777777778, places=2)

def test_train_save_load_predict_bootstrap_randomforest(self):

Expand All @@ -242,15 +242,15 @@ def test_train_save_load_predict_bootstrap_randomforest(self):

self.assertAlmostEqual(self.model.evaluate(xs, ys)['RMSE'], 0.17634739353518517, places=4)
self.assertAlmostEqual(self.model.evaluate_bagging(xs, ys)['RMSE'], 0.1734023350765026, places=4)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986, places=3)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333, places=1)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985, places=1)

self.assertAlmostEqual(loaded_model.evaluate(xs, ys)['RMSE'], 0.17634739353518517, places=4)
self.assertAlmostEqual(loaded_model.evaluate_bagging(xs, ys)['RMSE'], 0.1734023350765026, places=4)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.35136642421562986, places=3)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.4185833333333333, places=1)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.6789999999999985, places=1)

def test_train_save_load_predict_residue_bootstrap_libsvmnusvr(self):

Expand All @@ -267,15 +267,15 @@ def test_train_save_load_predict_residue_bootstrap_libsvmnusvr(self):

self.assertAlmostEqual(self.model.evaluate(xs, ys)['RMSE'], 0.6226308662005923, places=4)
self.assertAlmostEqual(self.model.evaluate_bagging(xs, ys)['RMSE'], 0.955069756575906, places=4)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.3437214386264522)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.4674811219156174)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.671073568422049)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.3437214386264522, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.4674811219156174, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.671073568422049, places=2)

self.assertAlmostEqual(loaded_model.evaluate(xs, ys)['RMSE'], 0.6226313987163097, places=4)
self.assertAlmostEqual(loaded_model.evaluate_bagging(xs, ys)['RMSE'], 0.9550696941149013, places=4)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.3437213993262196)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.4674811883034935)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.671073598975557)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.3437213993262196, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.4674811883034935, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.671073598975557, places=2)

def test_train_save_load_predict_residue_bootstrap_randomforest(self):

Expand All @@ -294,15 +294,15 @@ def test_train_save_load_predict_residue_bootstrap_randomforest(self):

self.assertAlmostEqual(self.model.evaluate(xs, ys)['RMSE'], 0.17634739353518517, places=4)
self.assertAlmostEqual(self.model.evaluate_bagging(xs, ys)['RMSE'], 0.2731966696149454, places=4)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.2176592325445535)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.621466666666666)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.4107916666666656)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_stddev'], 0.2176592325445535, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_low'], 3.621466666666666, places=2)
self.assertAlmostEqual(self.model.evaluate_stddev(xs)['mean_ci95_high'], 4.4107916666666656, places=2)

self.assertAlmostEqual(loaded_model.evaluate(xs, ys)['RMSE'], 0.17634739353518517, places=4)
self.assertAlmostEqual(loaded_model.evaluate_bagging(xs, ys)['RMSE'], 0.2731966696149454, places=4)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.2176592325445535)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.621466666666666)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.4107916666666656)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_stddev'], 0.2176592325445535, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_low'], 3.621466666666666, places=2)
self.assertAlmostEqual(loaded_model.evaluate_stddev(xs)['mean_ci95_high'], 4.4107916666666656, places=2)


if __name__ == '__main__':
Expand Down

0 comments on commit fe36d71

Please sign in to comment.