diff --git a/.gitignore b/.gitignore index e2a4fa3b..95022f46 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,4 @@ tests/cover/ tests/htmlcov/ .idea/* docs/_build/* +docs/tables/*.csv \ No newline at end of file diff --git a/.idea/.name b/.idea/.name deleted file mode 100644 index f06fb359..00000000 --- a/.idea/.name +++ /dev/null @@ -1 +0,0 @@ -pysd \ No newline at end of file diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml deleted file mode 100644 index 7fb8ed00..00000000 --- a/.idea/codeStyleSettings.xml +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/dictionaries/houghton.xml b/.idea/dictionaries/houghton.xml deleted file mode 100644 index fdb52ce8..00000000 --- a/.idea/dictionaries/houghton.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - houghton - vensim - - - \ No newline at end of file diff --git a/.idea/encodings.xml b/.idea/encodings.xml deleted file mode 100644 index 97626ba4..00000000 --- a/.idea/encodings.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml deleted file mode 100644 index 3bd5b081..00000000 --- a/.idea/inspectionProfiles/Project_Default.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml deleted file mode 100644 index 3b312839..00000000 --- a/.idea/inspectionProfiles/profiles_settings.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml deleted file mode 100644 index 845831c5..00000000 --- a/.idea/modules.xml +++ /dev/null @@ -1,8 +0,0 @@ - - - - - - - - \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml deleted file mode 100644 index 06b38168..00000000 --- a/.idea/vcs.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - - \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index b129dda9..2492107b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,4 @@ include requirements.txt include README.md include LICENSE -include pysd/translation/xmile/smile.grammar +graft pysd/translators/*/parsing_grammars diff --git a/README.md b/README.md index b2d93de7..a59b83cb 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ PySD ==== + [![Coverage Status](https://coveralls.io/repos/github/JamesPHoughton/pysd/badge.svg?branch=master)](https://coveralls.io/github/JamesPHoughton/pysd?branch=master) [![Anaconda-Server Badge](https://anaconda.org/conda-forge/pysd/badges/version.svg)](https://anaconda.org/conda-forge/pysd) [![PyPI version](https://badge.fury.io/py/pysd.svg)](https://badge.fury.io/py/pysd) @@ -13,16 +14,14 @@ Simulating System Dynamics Models in Python This project is a simple library for running [System Dynamics](http://en.wikipedia.org/wiki/System_dynamics) models in python, with the purpose of improving integration of *Big Data* and *Machine Learning* into the SD workflow. -**The current version needs to run at least Python 3.7. If you need support for Python 2, please use the release here: https://github.com/JamesPHoughton/pysd/releases/tag/LastPy2** - -**table2py feature was dropped in version 2.0.0, please use the release here if you want to build PySD model from a tabular file: https://github.com/JamesPHoughton/pysd/releases/tag/v1.11.0** +**The current version needs to run at least Python 3.7.** ### Resources + See the [project documentation](http://pysd.readthedocs.org/) for information about: - [Installation](http://pysd.readthedocs.org/en/latest/installation.html) -- [Basic Usage](http://pysd.readthedocs.org/en/latest/basic_usage.html) -- [Function Reference](http://pysd.readthedocs.org/en/latest/functions.html) +- [Getting Started](http://pysd.readthedocs.org/en/latest/getting_started.html) For standard methods for data analysis with SD models, see the [PySD Cookbook](https://github.com/JamesPHoughton/PySD-Cookbook), containing (for example): @@ -38,7 +37,6 @@ You can also cite the library using the [DOI provided by Zenodo](https://zenodo. [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.5654824.svg)](https://doi.org/10.5281/zenodo.5654824) - ### Why create a new SD simulation engine? There are a number of great SD programs out there ([Vensim](http://vensim.com/), [iThink](http://www.iseesystems.com/Softwares/Business/ithinkSoftware.aspx), [AnyLogic](http://www.anylogic.com/system-dynamics), [Insight Maker](http://insightmaker.com/), and [others](http://en.wikipedia.org/wiki/List_of_system_dynamics_software)). In order not to waste our effort, or fall victim to the [Not-Invented-Here](http://en.wikipedia.org/wiki/Not_invented_here) fallacy, we should have a very good reason for starting a new project. diff --git a/docs/Makefile b/docs/Makefile index 1529adff..4f8ee358 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -48,9 +48,12 @@ help: @echo " doctest to run all doctests embedded in the documentation (if enabled)" @echo " coverage to run coverage check of the documentation (if enabled)" -clean: +clean: clean_tables rm -rf $(BUILDDIR)/* +clean_tables: + rm -f tables/*.csv + html: $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html @echo diff --git a/docs/development/about.rst b/docs/about.rst similarity index 77% rename from docs/development/about.rst rename to docs/about.rst index bdb33818..05fa7636 100644 --- a/docs/development/about.rst +++ b/docs/about.rst @@ -8,25 +8,25 @@ The last few years have witnessed a massive growth in the collection of social a So far, however, these new techniques are largely confined to variants of statistical summary, categorization, and inference; and if causal models are used, they are generally static in nature, ignoring the dynamic complexity and feedback structures of the systems in question. As the field of data science matures, there will be increasing demand for insights beyond those available through analysis unstructured by causal understanding. At that point data scientists may seek to add dynamic models of system structure to their toolbox. -The field of system dynamics has always been interested in learning about social systems, and specializes in understanding dynamic complexity. There is likewise a long tradition of incorporating various forms of data into system dynamics models.3 While system dynamics practice has much to gain from the emergence of new volumes of social data, the community has yet to benefit fully from the data science revolution. +The field of system dynamics has always been interested in learning about social systems, and specializes in understanding dynamic complexity. There is likewise a long tradition of incorporating various forms of data into system dynamics models. While system dynamics practice has much to gain from the emergence of new volumes of social data, the community has yet to benefit fully from the data science revolution. There are a variety of reasons for this, the largest likely being that the two communities have yet to commingle to a great extent. A further, and ultimately more tractable reason is that the tools of system dynamics and the tools of data analytics are not tightly integrated, making joint method analysis unwieldy. There is a rich problem space that depends upon the ability of these fields to support one another, and so there is a need for tools that help the two methodologies work together. PySD is designed to meet this need. General approaches for integrating system dynamic models and data analytics --------------------------------------------------------------------------- -Before considering how system dynamics techniques can be used in data science applications, we should consider the variety of ways in which the system dynamics community has traditionally dealt with integration of data and models. +Before considering how system dynamics techniques can be used in data science applications, we should consider the variety of ways in which the system dynamics community has traditionally dealt with integration of data and models. The first paradigm for using numerical data in support of modeling efforts is to import data into system dynamics modeling software. Algorithms for comparing models with data are built into the tool itself, and are usable through a graphical front-end interface as with model fitting in Vensim, or through a programming environment unique to the tool. When new techniques such as Markov chain Monte Carlo analysis become relevant to the system dynamics community, they are often brought into the SD tool. - + This approach appropriately caters to system dynamics modelers who want to take advantage of well-established data science techniques without needing to learn a programming language, and extends the functionality of system dynamics to the basics of integrated model analysis. -A second category of tools uses a standard system dynamics tool as a computation engine for analysis performed in a coding environment. This is the approach taken by the Exploratory Modeling Analysis (EMA) Workbench6, or the Behavior Analysis and Testing Software (BATS)7. This first step towards bringing system dynamics to a more inclusive analysis environment enables many new types of model understanding, but imposes limits on the depth of interaction with models and the ability to scale simulation to support large analysis. +A second category of tools uses a standard system dynamics tool as a computation engine for analysis performed in a coding environment. This is the approach taken by the Exploratory Modeling Analysis (EMA) Workbench, or the Behavior Analysis and Testing Software (BATS). This first step towards bringing system dynamics to a more inclusive analysis environment enables many new types of model understanding, but imposes limits on the depth of interaction with models and the ability to scale simulation to support large analysis. + +A third category of tools imports the models created by traditional tools to perform analyses independently of the original modeling tool. An example of this is SDM-Doc, a model documentation tool, or Abdel-Gawad et. al.’s eigenvector analysis tool. It is this third category to which PySD belongs. -A third category of tools imports the models created by traditional tools to perform analyses independently of the original modeling tool. An example of this is SDM-Doc8, a model documentation tool, or Abdel-Gawad et. al.’s eigenvector analysis tool9. It is this third category to which PySD belongs. - The central paradigm of PySD is that it is more efficient to bring the mature capabilities of system dynamics into an environment in use for active development in data science, than to attempt to bring each new development in inference and machine learning into the system dynamics enclave. -PySD reads a model file – the product of a modeling program such as Vensim10 or Stella/iThink11 – and cross compiles it into Python, providing a simulation engine that can run these models natively in the Python environment. It is not a substitute for these tools, and cannot be used to replace a visual model construction environment. +PySD reads a model file – the product of a modeling program such as Vensim or Stella/iThink – and cross compiles it into Python, providing a simulation engine that can run these models natively in the Python environment. It is not a substitute for these tools, and cannot be used to replace a visual model construction environment. diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst index 0a15f010..8aadbea5 100644 --- a/docs/advanced_usage.rst +++ b/docs/advanced_usage.rst @@ -1,7 +1,7 @@ Advanced Usage ============== -The power of PySD, and its motivation for existence, is its ability to tie in to other models and analysis packages in the Python environment. In this section we’ll discuss how those connections happen. +The power of PySD, and its motivation for existence, is its ability to tie in to other models and analysis packages in the Python environment. In this section we discuss how those connections happen. Replacing model components with more complex objects @@ -16,11 +16,11 @@ However, when we made the room temperature something that varied with time, PySD def room_temperature(): return np.interp(t, series.index, series.values) -This drew on the internal state of the system, namely the time t, and the time-series data series that that we wanted to variable to represent. This process of substitution is available to the user, and we can replace functions ourselves, if we are careful. +This drew on the internal state of the system, namely the time t, and the time-series data series that we wanted the variable to represent. This process of substitution is available to the user, and we can replace functions ourselves, if we are careful. Because PySD assumes that all components in a model are represented as functions taking no arguments, any component that we wish to modify must be replaced with a function taking no arguments. As the state of the system and all auxiliary or flow methods are public, our replacement function can call these methods as part of its internal structure. -In our teacup example, suppose we didn’t know the functional form for calculating the heat lost to the room, but instead had a lot of data of teacup temperatures and heat flow rates. We could use a regression model (here a support vector regression from Scikit-Learn) in place of the analytic function:: +In our teacup example, suppose we did not know the functional form for calculating the heat lost to the room, but instead had a lot of data of teacup temperatures and heat flow rates. We could use a regression model (here a support vector regression from Scikit-Learn) in place of the analytic function:: from sklearn.svm import SVR regression = SVR() @@ -35,22 +35,22 @@ Once the regression model is fit, we write a wrapper function for its predict me room_temp = model.components.room_temperature() return regression.predict([room_temp, tea_temp])[0] -In order to substitute this function directly for the heat_loss_to_room model component using the :py:func:`set_component()` method:: +To substitute this function directly for the heat_loss_to_room model component using the :py:meth:`.set_components` method:: model.set_components({'heat_loss_to_room': new_heatflow_function}) -If you want to replace a subscripted variable, you need to ensure that the output from the new function is the same as the previous one. You can check the current coordinates and dimensions of a component by using :py:data:`.get_coords(variable_name)` as it is explained in :doc:`basic usage <../basic_usage>`. +If you want to replace a subscripted variable, you need to ensure that the output from the new function is the same as the previous one. You can check the current coordinates and dimensions of a component by using :py:meth:`.get_coords` as it is explained in :doc:`Getting started <../getting_started>`. .. note:: Alternatively, you can also set a model component directly:: model.components.heat_loss_to_room = new_heatflow_function - However, this will only accept the python name of the model component. While for the :py:func:`set_component()` method, the original name can be also used. + However, this will only accept the python name of the model component. While for the :py:meth:`.set_components` method, the original name can be also used. Splitting Vensim views in separate Python files (modules) --------------------------------------------------------- -In order to replicate the Vensim views in translated models, the user can set the `split_views` argument to True in the :py:func:`read_vensim` function:: +In order to replicate the Vensim views in the translated models, the user can set the `split_views` argument to True in the :py:func:`pysd.read_vensim` function:: read_vensim("many_views_model.mdl", split_views=True) @@ -65,12 +65,10 @@ In a Vensim model with three separate views (e.g. `view_1`, `view_2` and `view_3 | │ ├── view_1.py | │ ├── view_2.py | │ └── view_3.py -| ├── _namespace_many_views_model.json | ├── _subscripts_many_views_model.json -| ├── _dependencies_many_views_model.json | ├── many_views_model.py -| -| + + .. note :: Often, modelers wish to organise views further. To that end, a common practice is to include a particular character in the View name to indicate that what comes after it is the name of the subview. For instance, we could name one view as `ENERGY.Supply` and another one as `ENERGY.Demand`. @@ -78,19 +76,19 @@ In a Vensim model with three separate views (e.g. `view_1`, `view_2` and `view_3 read_vensim("many_views_model.mdl", split_views=True, subview_sep=["."]) -If macros are present, they will be self-contained in files named as the macro itself. The macro inner variables will be placed inside the module that corresponds with the view in which they were defined. +If macros are present, they will be self-contained in files named after the macro itself. The macro inner variables will be placed inside the module that corresponds with the view in which they were defined. Starting simulations from an end-state of another simulation ------------------------------------------------------------ -The current state of a model can be saved in a pickle file using the :py:data:`.export()` method:: +The current state of a model can be saved in a pickle file using the :py:meth:`.export` method:: import pysd model1 = pysd.read_vensim("my_model.mdl") model1.run(final_time=50) model1.export("final_state.pic") -Then the exported data can be used in another session:: +then the exported data can be used in another session:: import pysd model2 = pysd.load("my_model.py") @@ -105,21 +103,22 @@ the new simulation will have initial time equal to 50 with the saved values from model1.run(final_time=50, return_timestamps=[]) .. note:: - The changes done with *params* arguments are not ported to the new model (*model2*) object that you initialize with *final_state.pic*. If you want to keep them, you need to call run with the same *params* values as in the original model (*model1*). + The changes made with the *params* arguments are not ported to the new model (*model2*) object that you initialize with *final_state.pic*. If you want to keep them, you need to call run with the same *params* values as in the original model (*model1*). .. warning:: - Exported data is saved and loaded using `pickle `_, this data can be incompatible with future versions of - *PySD* or *xarray*. In order to prevent data losses save always the source code. + Exported data is saved and loaded using `pickle `_. The data stored in the pickles may be incompatible with future versions of + *PySD* or *xarray*. In order to prevent data losses, always save the source code. Selecting and running a submodel -------------------------------- -A submodel of a translated model can be selected in order to run only a part of the original model. This can be done through the :py:data:`.select_submodel()` method: +A submodel of a translated model can be run as a standalone model. This can be done through the :py:meth:`.select_submodel` method: + +.. automethod:: pysd.py_backend.model.Model.select_submodel + :noindex: -.. autoclass:: pysd.py_backend.statefuls.Model - :members: select_submodel -In order to preview the needed exogenous variables the :py:data:`.get_dependencies()` method can be used: +In order to preview the needed exogenous variables, the :py:meth:`.get_dependencies` method can be used: -.. autoclass:: pysd.py_backend.statefuls.Model - :members: get_dependencies +.. automethod:: pysd.py_backend.model.Model.get_dependencies + :noindex: diff --git a/docs/basic_usage.rst b/docs/basic_usage.rst deleted file mode 100644 index 2cb72a85..00000000 --- a/docs/basic_usage.rst +++ /dev/null @@ -1,249 +0,0 @@ -Basic Usage -=========== - -Importing a model and getting started -------------------------------------- -To begin, we must first load the PySD module, and use it to import a supported model file:: - - >>> import pysd - >>> model = pysd.read_vensim('Teacup.mdl') - - -This code creates an instance of the PySD class loaded with an example model that we will use as the system dynamics equivalent of ‘Hello World’: a cup of tea cooling to room temperature. - -.. image:: images/Teacup.png - :width: 350 px - :align: center - -.. note:: - The teacup model can be found in the `samples of the test-models repository `_. - -To view a synopsis of the model equations and documentation, call the :py:func:`.doc()` method of the model class. This will generate a listing of all the model elements, their documentation, units, equations, and initial values, where appropriate, and return them as a :py:class:`pandas.DataFrame`. Here is a sample from the teacup model:: - - >>> model.doc() - - Real Name Py Name Unit Lims Type Subs Eqn Comment - 0 Characteristic Time characteristic_time Minutes (0.0, None) constant None 10 How long will it take the teacup to cool 1/e o... - 1 FINAL TIME final_time Minute (None, None) constant None 30 The final time for the simulation. - 2 Heat Loss to Room heat_loss_to_room Degrees Fahrenheit/Minute (None, None) component None (Teacup Temperature - Room Temperature) / Char... This is the rate at which heat flows from the ... - 3 INITIAL TIME initial_time Minute (None, None) constant None 0 The initial time for the simulation. - 4 Room Temperature room_temperature Degrees Fahrenheit (-459.67, None) constant None 70 Put in a check to ensure the room temperature ... - 5 SAVEPER saveper Minute (0.0, None) component None TIME STEP The frequency with which output is stored. - 6 TIME STEP time_step Minute (0.0, None) constant None 0.125 The time step for the simulation. - 7 Teacup Temperature teacup_temperature Degrees Fahrenheit (32.0, 212.0) component None INTEG ( -Heat Loss to Room, 180) The model is only valid for the liquid phase o... - - -.. note:: - You can also load an already translated model file, what will be faster as you will load a Python file:: - - >>> import pysd - >>> model = pysd.load('Teacup.py') - -.. note:: - The functions :py:func:`read_vensim()`, :py:func:`read_xmile()` and :py:func:`load()` have optional arguments for advanced usage, you can check the full description in :doc:`User Functions Reference <../functions>` or using :py:func:`help()` e.g.:: - - >>> import pysd - >>> help(pysd.load) - - -Running the Model ------------------ -The simplest way to simulate the model is to use the :py:func:`.run()` command with no options. This runs the model with the default parameters supplied by the model file, and returns a :py:class:`pandas.DataFrame` of the values of the model components at every timestamp:: - - >>> stocks = model.run() - >>> stocks - - Characteristic Time Heat Loss to Room Room Temperature Teacup Temperature FINAL TIME INITIAL TIME SAVEPER TIME STEP - 0.000 10 11.000000 70 180.000000 30 0 0.125 0.125 - 0.125 10 10.862500 70 178.625000 30 0 0.125 0.125 - 0.250 10 10.726719 70 177.267188 30 0 0.125 0.125 - 0.375 10 10.592635 70 175.926348 30 0 0.125 0.125 - 0.500 10 10.460227 70 174.602268 30 0 0.125 0.125 - ... ... ... ... ... ... ... ... ... - 29.500 10 0.565131 70 75.651312 30 0 0.125 0.125 - 29.625 10 0.558067 70 75.580671 30 0 0.125 0.125 - 29.750 10 0.551091 70 75.510912 30 0 0.125 0.125 - 29.875 10 0.544203 70 75.442026 30 0 0.125 0.125 - 30.000 10 0.537400 70 75.374001 30 0 0.125 0.125 - -[241 rows x 8 columns] - -Pandas gives us simple plotting capability, so we can see how the cup of tea behaves:: - - >>> import matplotlib.pyplot as plt - >>> stocks["Teacup Temperature"].plot() - >>> plt.title("Teacup Temperature") - >>> plt.ylabel("Degrees F") - >>> plt.xlabel("Minutes") - >>> plt.grid() - -.. image:: images/Teacup_Cooling.png - :width: 400 px - :align: center - -To show a progressbar during the model integration the progress flag can be passed to the :py:func:`.run()` command, progressbar package is needed:: - - >>> stocks = model.run(progress=True) - -Running models with DATA type components -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Venim's regular DATA type components are given by an empty expression in the model equation. These values are read from a binary `.vdf` file. PySD allows running models with this kind of data definition using the data_files argument when calling :py:func:`.run()` command, e.g.:: - - >>> stocks = model.run(data_files="input_data.tab") - -Several files can be passed by using a list, then if the data information has not been found in the first file, the next one will be used until finding the data values:: - - >>> stocks = model.run(data_files=["input_data.tab", "input_data2.tab", ..., "input_datan.tab"]) - -If a variable is given in different files to choose the specific file a dictionary can be used:: - - >>> stocks = model.run(data_files={"input_data.tab": ["data_var1", "data_var3"], "input_data2.tab": ["data_var2"]}) - -.. note:: - Only `tab` and `csv` files are supported, they should be given as a table, each variable one column (or row) and the time in the first column (or first row). The column (or row) names can be given using the original name or using python names. - -.. note:: - Subscripted variables must be given in the vensim format, one column (or row) per subscript combination. Example of column names for 2x2 variable: - `subs var[A, C]` `subs var[B, C]` `subs var[A, D]` `subs var[B, D]` - -Outputting various run information ----------------------------------- -The :py:func:`.run()` command has a few options that make it more useful. In many situations we want to access components of the model other than merely the stocks – we can specify which components of the model should be included in the returned dataframe by including them in a list that we pass to the :py:func:`.run()` command, using the return_columns keyword argument:: - - >>> model.run(return_columns=['Teacup Temperature', 'Room Temperature']) - - Teacup Temperature Room Temperature - 0.000 180.000000 70 - 0.125 178.625000 70 - 0.250 177.267188 70 - 0.375 175.926348 70 - 0.500 174.602268 70 - ... ... ... - 29.500 75.651312 70 - 29.625 75.580671 70 - 29.750 75.510912 70 - 29.875 75.442026 70 - 30.000 75.374001 70 - - [241 rows x 2 columns] - - -If the measured data that we are comparing with our model comes in at irregular timestamps, we may want to sample the model at timestamps to match. The :py:func:`.run()` function gives us this ability with the return_timestamps keyword argument:: - - >>> model.run(return_timestamps=[0, 1, 3, 7, 9.5, 13, 21, 25, 30]) - - Characteristic Time Heat Loss to Room Room Temperature Teacup Temperature FINAL TIME INITIAL TIME SAVEPER TIME STEP - 0.0 10 11.000000 70 180.000000 30 0 0.125 0.125 - 1.0 10 9.946940 70 169.469405 30 0 0.125 0.125 - 3.0 10 8.133607 70 151.336071 30 0 0.125 0.125 - 7.0 10 5.438392 70 124.383922 30 0 0.125 0.125 - 9.5 10 4.228756 70 112.287559 30 0 0.125 0.125 - 13.0 10 2.973388 70 99.733876 30 0 0.125 0.125 - 21.0 10 1.329310 70 83.293098 30 0 0.125 0.125 - 25.0 10 0.888819 70 78.888194 30 0 0.125 0.125 - 30.0 10 0.537400 70 75.374001 30 0 0.125 0.125 - - -Retrieving totally flat dataframe ---------------------------------- -The subscripted variables, in general, will be returned as *xarray.DataArray*s in the output *pandas.DataFrame*. To get a totally flat dataframe, like Vensim outuput the `flatten=True` when calling the run function:: - - >>> model.run(flatten=True) - -Setting parameter values ------------------------- -In many cases, we want to modify the parameters of the model to investigate its behavior under different assumptions. There are several ways to do this in PySD, but the :py:func:`.run()` function gives us a convenient method in the params keyword argument. - -This argument expects a dictionary whose keys correspond to the components of the model. The associated values can either be a constant, or a Pandas series whose indices are timestamps and whose values are the values that the model component should take on at the corresponding time. For instance, in our model we can set the room temperature to a constant value:: - - >>> model.run(params={'Room Temperature': 20}) - -Alternately, if we believe the room temperature is changing over the course of the simulation, we can give the run function a set of time-series values in the form of a Pandas series, and PySD will linearly interpolate between the given values in the course of its integration:: - - >>> import pandas as pd - >>> temp = pd.Series(index=range(30), data=range(20, 80, 2)) - >>> model.run(params={'Room Temperature': temp}) - -If the parameter value to change is a subscripted variable (vector, matrix...), there are three different options to set new value. Suposse we have ‘Subscripted var’ with dims :py:data:`['dim1', 'dim2']` and coordinates :py:data:`{'dim1': [1, 2], 'dim2': [1, 2]}`. A constant value can be used and all the values will be replaced:: - - >>> model.run(params={'Subscripted var': 0}) - -A partial *xarray.DataArray* can be used, for example a new variable with ‘dim2’ but not ‘dim2’, the result will be repeated in the remaining dimensions:: - - >>> import xarray as xr - >>> new_value = xr.DataArray([1, 5], {'dim2': [1, 2]}, ['dim2']) - >>> model.run(params={'Subscripted var': new_value}) - -Same dimensions *xarray.DataArray* can be used (recommended):: - - >>> import xarray as xr - >>> new_value = xr.DataArray([[1, 5], [3, 4]], {'dim1': [1, 2], 'dim2': [1, 2]}, ['dim1', 'dim2']) - >>> model.run(params={'Subscripted var': new_value}) - -In the same way, a Pandas series can be used with constan values, partially defined *xarray.DataArrays* or same dimensions *xarray.DataArrays*. - -.. note:: - That once parameters are set by the run command, they are permanently changed within the model. We can also change model parameters without running the model, using PySD’s :py:data:`set_components(params={})` method, which takes the same params dictionary as the run function. We might choose to do this in situations where we’ll be running the model many times, and only want to spend time setting the parameters once. - -.. note:: - If you need to know the dimensions of a variable, you can check them by using :py:data:`.get_coords(variable__name)` function:: - - >>> model.get_coords('Room Temperature') - - None - - >>> model.get_coords('Subscripted var') - - ({'dim1': [1, 2], 'dim2': [1, 2]}, ['dim1', 'dim2']) - - this will return the coords dictionary and the dimensions list if the variable is subscripted or ‘None’ if the variable is an scalar. - -.. note:: - If you change the value of a lookup function by a constant, the constant value will be used always. If a *pandas.Series* is given the index and values will be used for interpolation when the function is called in the model, keeping the arguments that are included in the model file. - - If you change the value of any other variable type by a constant, the constant value will be used always. If a *pandas.Series* is given the index and values will be used for interpolation when the function is called in the model, using the time as argument. - - If you need to know if a variable takes arguments, i.e., if it is a lookup variable, you can check it by using :py:data:`.get_args(variable__name)` function:: - - >>> model.get_args('Room Temperature') - - [] - - >>> model.get_args('Growth lookup') - - ['x'] - -Setting simulation initial conditions -------------------------------------- -Finally, we can set the initial conditions of our model in several ways. So far, we’ve been using the default value for the initial_condition keyword argument, which is ‘original’. This value runs the model from the initial conditions that were specified originally by the model file. We can alternately specify a tuple containing the start time and a dictionary of values for the system’s stocks. Here we start the model with the tea at just above freezing:: - - >>> model.run(initial_condition=(0, {'Teacup Temperature': 33})) - -The new value setted can be a *xarray.DataArray* as it is explained in the previous section. - -Additionally we can run the model forward from its current position, by passing the initial_condition argument the keyword ‘current’. After having run the model from time zero to thirty, we can ask the model to continue running forward for another chunk of time:: - - >>> model.run(initial_condition='current', - return_timestamps=range(31, 45)) - -The integration picks up at the last value returned in the previous run condition, and returns values at the requested timestamps. - -There are times when we may choose to overwrite a stock with a constant value (ie, for testing). To do this, we just use the params value, as before. Be careful not to use 'params' when you really mean to be setting the initial condition! - - -Querying current values ------------------------ -We can easily access the current value of a model component using curly brackets. For instance, to find the temperature of the teacup, we simply call:: - - >>> model['Teacup Temperature'] - -If you try to get the current values of a lookup variable the previous method will fail as lookup variables take arguments. However, it is possible to get the full series of a lookup or data object with :py:func:`.get_series_data` method:: - - >>> model.get_series_data('Growth lookup') - -Supported functions -------------------- - -Vensim functions include: - -.. include:: development/supported_vensim_functions.rst diff --git a/docs/command_line_usage.rst b/docs/command_line_usage.rst index d7ff8966..385a44e7 100644 --- a/docs/command_line_usage.rst +++ b/docs/command_line_usage.rst @@ -4,14 +4,14 @@ Command Line Usage Basic command line usage ------------------------ -Most of the features available in :doc:`basic usage <../basic_usage>` are also available using command line. Running: +Most of the features available in :doc:`Getting started <../getting_started>` are also available using the command line. Running: .. code-block:: text python -m pysd Teacup.mdl -will translate *Teacup.mdl* to *Teacup.py* and run it with the default values. The output will be saved in *Teacup_output_%Y_%m_%d-%H_%M_%S_%f.tab*. The command line accepts several arguments, this can be checked using the *-h/--help* argument: +will translate *Teacup.mdl* to *Teacup.py* and run it with the default values. The output will be saved in *Teacup_output_%Y_%m_%d-%H_%M_%S_%f.tab*. The command line interface accepts several arguments, this can be checked using the *-h/--help* argument: .. code-block:: text @@ -19,7 +19,7 @@ will translate *Teacup.mdl* to *Teacup.py* and run it with the default values. T Set output file ^^^^^^^^^^^^^^^ -In order to set the output file *-o/--output-file* argument can be used: +In order to set the output file path, the *-o/--output-file* argument can be used: .. code-block:: text @@ -29,13 +29,13 @@ In order to set the output file *-o/--output-file* argument can be used: The output file can be a *.csv* or *.tab*. .. note:: - If *-o/--output-file* is not given the output will be saved in a file - that starts with the model file name and has a time stamp to avoid + If *-o/--output-file* is not given, the output will be saved in a file + that starts with the model file name followed by a time stamp to avoid overwritting files. Activate progress bar ^^^^^^^^^^^^^^^^^^^^^ -The progress bar can be activated using *-p/--progress* command: +The progress bar can be activated using the *-p/--progress* argument: .. code-block:: text @@ -44,9 +44,9 @@ The progress bar can be activated using *-p/--progress* command: Translation options ------------------- -Only translate model file -^^^^^^^^^^^^^^^^^^^^^^^^^ -To only translate the model file, it does not run the model, *-t/--trasnlate* command is provided: +Only translate the model file +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +To translate the model file and not run the model, the *-t/--trasnlate* command is provided: .. code-block:: text @@ -54,24 +54,25 @@ To only translate the model file, it does not run the model, *-t/--trasnlate* co Splitting Vensim views in different files ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -In order to split the Vensim model views in different files as explained in :doc:`advanced usage <../advanced_usage>`: +In order to split the Vensim model views in different files, as explained in :doc:`advanced usage <../advanced_usage>`, use the *--split-views* argument: .. code-block:: text - python -m pysd --split-modules many_views_model.mdl + python -m pysd --split-views many_views_model.mdl + Outputting various run information ---------------------------------- -The output number of variables can be modified bu passing them as arguments separated by commas, using *-r/return_columns* argument: +The number of output variables can be modified by passing them as arguments separated by commas, using the *-r/return_columns* argument: .. code-block:: text python -m pysd -r 'Teacup Temperature, Room Temperature' Teacup.mdl -Note that the argument passed after *-r/return_columns* should be inside '' to be properly read. Moreover each variable name must be split with commas. +Note that the a single string must be passed after the *-r/return_columns* argument, containing the names of the variables separated by commas. -Sometimes, the variable names have special characteres, such as commas, which can happen when trying to return a variable with subscripts. -In this case whe can save a *.txt* file with one variable name per row and use it as an argument: +Sometimes, variable names have special characteres, such as commas, which can happen when trying to return a variable with subscripts. +In this case we can save a *.txt* file with one variable name per row and use it as an argument: .. code-block:: text @@ -89,15 +90,15 @@ In this case whe can save a *.txt* file with one variable name per row and use i where *N* is an integer. .. note:: - The time outputs can be also modified using the model control variables, explained in next section. + The time outputs can also be modified using the model control variables, explained in next section. Modify model variables ---------------------- Modify model control variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -The model control variables such as the *initial time*. *final time*, *time step* and *saving step* can be easily -modified using the *-I/--initial_time*, *-F/--final-time*, *-T/--time-step* and *-S/--saveper* commands respectively. For example: +The values of the model control variables (i.e. *initial time*. *final time*, *time step* and *saving step*) can be +modified using the *-I/--initial_time*, *-F/--final-time*, *-T/--time-step* and *-S/--saveper* arguments, respectively. For example: .. code-block:: text @@ -106,32 +107,30 @@ modified using the *-I/--initial_time*, *-F/--final-time*, *-T/--time-step* and will set the initial time to 2005, the final time to 2010 and the time step to 1. .. note:: - If *-R/--return-timestamps* argument is used the *final time* and *saving step* will be ignored. + If the *-R/--return-timestamps* argument is used, the *final time* and *saving step* will be ignored. Modify model variables ^^^^^^^^^^^^^^^^^^^^^^ -In order to modify the values of model variables they can be passed after the model file: +To modify the values of model variables, their new values may be passed after the model file: .. code-block:: text python -m pysd Teacup.mdl 'Room Temperature'=5 -this will set *Room Temperature* variable to the constant value 5. A series can be also passed -to change a value of a value to a time dependent series or the interpolation values -of a lookup variable two lists of the same length must be given: +this will set *Room Temperature* variable to 5. A time series or a lookup can also be passed +as the new value of a variable as two lists of the same length: .. code-block:: text python -m pysd Teacup.mdl 'Temperature Lookup=[[1, 2, 3, 4], [10, 15, 17, 18]]' -The first list will be used for the *time* or *x* values and the second for the data. See setting parameter values in :doc:`basic usage <../basic_usage>` for more information. +The first list will be used for the *time* or *x* values, and the second for the data values. See setting parameter values in :doc:`Getting started <../getting_started>` for further details. .. note:: - If a variable name or the right hand side are defined with whitespaces - it is needed to add '' define it, as has been done in the last example. + If a variable name or the right hand side are defined with white spaces, they must be enclosed in quotes, as in the previous example. Several variables can be changed at the same time, e.g.: @@ -141,7 +140,7 @@ Several variables can be changed at the same time, e.g.: Modify initial conditions of model variables ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Sometimes we do not want to change a variable value to a constant but change its initial value, for example change initial value of a stock object, this can be similarly done to the previos case but using ':' instead of '=': +Sometimes we do not want to change the actual value of a variable but we want to change its initial value instead. An example of this would be changing the initial value of a stock object. This can be done similarly to what was shown in the previos case, but using ':' instead of '=': .. code-block:: text @@ -149,12 +148,12 @@ Sometimes we do not want to change a variable value to a constant but change its this will set initial *Teacup Temperature* to 30. -Putting It All Together +Putting it all together ----------------------- -Several commands can be used together, first need to add optional arguments, those starting with '-', next the model file, and last the variable or variables to change, for example: +Several commands may be used together. The optional arguments and model arguments go first (those starting with '-' or '--'), then the model file path, and finally the variable or variables to change: .. code-block:: text python -m pysd -o my_output_file.csv --progress --final-time=2010 --time-step=1 Teacup.mdl 'Room Temperature'=5 temperature_lookup='[[1, 2, 3, 4], [10, 15, 17, 18]]' 'Teacup Temperature':30 -will save step 1 outputs until 2010 in *my_output_file.csv*, showing a progressbar during integration and settung foo to *5* and *temperature_lookup* to ((1, 10), (2, 15), (3, 17), (4, 18)) and initial *Teacup Temperature* to 30. \ No newline at end of file +will save step 1 outputs until 2010 in *my_output_file.csv*, showing a progressbar during integration and setting foo to *5*, *temperature_lookup* to ((1, 10), (2, 15), (3, 17), (4, 18)) and initial *Teacup Temperature* to 30. \ No newline at end of file diff --git a/docs/development/complement.rst b/docs/complement.rst similarity index 73% rename from docs/development/complement.rst rename to docs/complement.rst index fd1253ac..c9f958c2 100644 --- a/docs/development/complement.rst +++ b/docs/complement.rst @@ -3,8 +3,12 @@ Complementary Projects The most valuable component for better integrating models with *basically anything else* is a standard language for communicating the structure of those models. That language is `XMILE `_. The draft specifications for this have been finalized and the standard should be approved in the next few months. -A python library for analyzing system dynamics models called the `Exploratory Modeling and Analysis (EMA) Workbench `_ is being developed by `Erik Pruyt `_ and `Jan Kwakkel `_ at TU Delft. This package implements a variety of analysis methods that are unique to dynamic models, and could work very tightly with PySD. +A Python library for analyzing system dynamics models called the `Exploratory Modeling and Analysis (EMA) Workbench `_ is being developed by `Erik Pruyt `_ and `Jan Kwakkel `_ at TU Delft. This package implements a variety of analysis methods that are unique to dynamic models, and could work very tightly with PySD. -An excellent javascript library called `sd.js `_ created by Bobby Powers at `SDlabs `_ exists as a standalone SD engine, and provides a beautiful front end. This front end could be rendered as an iPython widget to facilitate display of SD models. +An excellent JavaScript library called `sd.js `_ created by Bobby Powers at `SDlabs `_ exists as a standalone SD engine, and provides a beautiful front end. This front end could be rendered as an iPython widget to facilitate display of SD models. -The `Behavior Analysis and Testing Software(BATS) `_ delveloped by `Gönenç Yücel `_ includes a really neat method for categorizing behavior modes and exploring parameter space to determine the boundaries between them. \ No newline at end of file +The `Behavior Analysis and Testing Software(BATS) `_ delveloped by `Gönenç Yücel `_ includes a really neat method for categorizing behavior modes and exploring parameter space to determine the boundaries between them. + +The `SDQC library `_ developed by Eneko Martin Martinez may be used to check the quality of the data imported by Vensim models from speadsheet files. + +The `excels2vensim library `_, also developed by Eneko Martin Martinez, aims to simplify the incorporation of equations from external data into Vensim. \ No newline at end of file diff --git a/docs/conf.py b/docs/conf.py index c39e52c6..295fe981 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -20,11 +20,16 @@ sys.path.insert(0, os.path.abspath('../')) +from docs.generate_tables import generate_tables + + +# Generate tables used for documentation +generate_tables() MOCK_MODULES = [ 'numpy', 'scipy', 'matplotlib', 'matplotlib.pyplot', 'scipy.stats', 'scipy.integrate', 'pandas', 'parsimonious', 'parsimonious.nodes', - 'lxml', 'xarray', 'autopep8', 'scipy.linalg', 'parsimonious.exceptions', + 'xarray', 'autopep8', 'scipy.linalg', 'parsimonious.exceptions', 'scipy.stats.distributions', 'progressbar', 'black' ] @@ -48,9 +53,15 @@ 'sphinx.ext.mathjax', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', - 'sphinx.ext.intersphinx' + 'sphinx.ext.intersphinx', + "sphinx.ext.extlinks" ] +extlinks = { + "issue": ("https://github.com/JamesPHoughton/pysd/issues/%s", "issue #%s"), + "pull": ("https://github.com/JamesPHoughton/pysd/pull/%s", "PR #%s"), +} + # Add any paths that contain templates here, relative to this directory. templates_path = ['_templates'] @@ -138,5 +149,10 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3.7', None), 'pysdcookbook': ('http://pysd-cookbook.readthedocs.org/en/latest/', None), - 'pandas': ('https://pandas.pydata.org/docs/', None) + 'pandas': ('https://pandas.pydata.org/docs/', None), + 'xarray': ('https://docs.xarray.dev/en/stable/', None), + 'numpy': ('https://numpy.org/doc/stable/', None) } + +# -- Options for autodoc -------------------------------------------------- +autodoc_member_order = 'bysource' diff --git a/docs/development/Building a SMILE to Python Translator using Parsimonious.ipynb b/docs/development/Building a SMILE to Python Translator using Parsimonious.ipynb deleted file mode 100644 index 72846c2f..00000000 --- a/docs/development/Building a SMILE to Python Translator using Parsimonious.ipynb +++ /dev/null @@ -1,1070 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "e8d6871b", - "metadata": {}, - "source": [ - "#Building a SMILE to Python Translator\n", - "\n", - "[SMILE](http://www.iseesystems.com/community/support/SMILEv4.pdf) is the language description used in the XMILE format. Part of parsing XMILE files will be to parse strings of code in SMILE format. To do this we need to understand the SMILE grammar - and more importantly, python needs to know how to do it as well.\n", - "\n", - "In this notebook we'll be using [parsimonious](https://github.com/erikrose/parsimonious) to interpret our grammar, parse our strings, and return for us an [Abstract Syntax Tree](). There are a variety of other tools we could use:\n", - "\n", - "- [PLY](http://www.dabeaz.com/ply/) - 55397 downloads in the last month\n", - "- [plex](https://pythonhosted.org/plex/) - 949 downloads in the last month\n", - "- [tokenizertools](https://github.com/dbc/tokenizertools) - 642 downloads in the last month\n", - "- [pyparsing](http://pyparsing.wikispaces.com/) - 221150 downloads in the last month\n", - "- [ANTLR](https://github.com/antlr/antlr4) - not python native\n", - "- [others](https://github.com/erikrose/mediawiki-parser/blob/master/parsers.rst)\n", - "\n", - "Parsimonious seems to strike a good ballance between new, high-level-functionality, and maturity.\n", - "\n", - "We will use [Parsing Expression Grammar](http://en.wikipedia.org/wiki/Parsing_expression_grammar) to specify how parsimonious should interpret our input strings. \n", - "Here is a good [slide deck](https://ece.uwaterloo.ca/~vganesh/TEACHING/W2014/lectures/lecture16.pdf) of how PEG works,\n", - "here is the original [paper](http://www.brynosaurus.com/pub/lang/peg.pdf) describing the concept,\n", - "and here are some [reasonable](https://github.com/PhilippeSigaud/Pegged/wiki/PEG-Basics)\n", - "[tutorials](http://nathansuniversity.com/pegs.html)\n", - "\n", - "PEG compares to a few other syntaxes for describing grammar:\n", - "\n", - "- [BNF](http://en.wikipedia.org/wiki/Backus%E2%80%93Naur_Form)\n", - "- [EBNF](http://en.wikipedia.org/wiki/Extended_Backus%E2%80%93Naur_Form)\n", - "\n", - "Here are some examples of \n", - "[parsimonious](http://nullege.com/codes/show/src@c@s@csp-validator-0.2@csp_validator@csp.py/5/parsimonious.grammar.Grammar/python)\n", - "in [action](http://jeffrimko.blogspot.com/2013/05/parsing-with-parsimonious.html)\n", - "\n", - "Parsimonious has its own spin on PEG (mostly replacing `<-` with `=`) and those changes are listed on the main \n", - "[github page](https://github.com/erikrose/parsimonious).\n", - "\n", - "\n", - "We're just building a translator, but if we wanted to build a full-out interpreter, here is how we should do it:\n", - "\n", - "- Described in a [video](https://www.youtube.com/watch?v=1h1mM7VwNGo)\n", - "- [Code](https://github.com/halst/mini/blob/master/mini.py) and [Tests](https://github.com/halst/mini/blob/master/test_mini.py) shared here\n" - ] - }, - { - "cell_type": "markdown", - "id": "b1002ca0", - "metadata": {}, - "source": [ - "### So, in general, how does this work?\n", - "\n", - "The parser looks at the first line, and tries to match it. If the first line fails, the whole thing fails. \n", - "\n", - "Regular expressions are included with the syntax `~\"expression\"`\n", - "\n", - "Statements that include `a / b / etc...` give you the preferential choice for the string element to be of type `a`, and if not, then perhaps `b`, and so on.\n", - "\n", - "As with regular expressions, a trailing +, ?, or * denotes the number of times the preceeding pattern should be matched.\n", - "\n", - "\n", - "For example, in this grammar:\n", - "\n", - " grammar = \"\"\"\n", - " Term = Factor Additive*\n", - " Additive= (\"+\"/\"-\") Factor\n", - "\n", - " Factor = Primary Multiplicative*\n", - " Multiplicative = (\"*\" / \"/\") Primary\n", - "\n", - " Primary = Parens / Neg / Number \n", - " Parens = \"(\" Term \")\"\n", - " Neg = \"-\" Primary\n", - " Number = ~\"[0-9]+\"\n", - " \"\"\"\n", - " \n", - "if we try and parse \"5+3\", then the parser looks at the first line `Term` and says: 'If this is going to match, then the first component needs to be a `Factor`', so it then goes and looks at the definition for `Factor` and says: 'If this is going to match, then the first component needs to be a `Primary`'. Then it goes to look at the definition for `Primary` and says: 'This might be a `Parens`, lets check. Then it goes and looks at the definition of `Parens` and finds that the first element does not equal to 5, so it says 'nope!' and goes back up a level to `Primary`. \n", - "\n", - "It then checks to see if the first component of the string fits the `Neg` pattern, and discovers that it doesn't, and returns to the `Primary` definition and checks the third option: `Number`. It goes to the definition of number and says 'Hey, at least the first character matches `Number` - but number asks for one or more characters between 0 and 9, so lets check the next character - it is a `+`, so that doesnt fit the pattern, so we'll capture 5 as a `Number`, then return up to `Primary` - and as there are no other commands listed in `Primary` also return to `Factor`. \n", - "\n", - "Now, factor asks for zero or more `Multiplicative` components, so lets check if our string (now with the 5 removed) matches the `Multiplicative` pattern. The first element of a multiplicative component should be '*', or '\\/', and it isnt, so lets pop back up to `Factor`, and then to `Term`.\n", - "\n", - "The term then goes on to see if the string (starting at '+') matches the additive pattern - and it sees that the '+' matches its first condition, and then goes on to check for a `Factor`, beginning with the '5' in the string. This follows the same path as we saw before to match the 5 as a factor element.\n", - "\n", - "The parser collects all of the components into a tree and returns it to the user." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "f69bad3f", - "metadata": {}, - "outputs": [], - "source": [ - "import parsimonious" - ] - }, - { - "cell_type": "markdown", - "id": "015fbb0e", - "metadata": {}, - "source": [ - "## Start with someone else's arithmetic grammar\n", - "by [Philippe Sigaud](), available [here]()\n", - "\n", - "This is a good example of how to get around the left-hand recursion issue.\n", - "\n", - "Our translator will have several parts, that we can see here.\n", - "\n", - "1. First, we define the grammar and compile it\n", - "2. Then we define a function to parse the Abstract Syntax Tree and translate any of its components (here translation is just to return a stringified version)\n", - "3. Then we parse the string we're interest in translating to an AST\n", - "4. Finally, we crawl the AST, compiling an output string." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "b3da2ede", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "5" - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#define the grammar\n", - "grammar = \"\"\"\n", - "Term = Factor (Add / Sub)*\n", - "Add = \"+\" Factor\n", - "Sub = \"-\" Factor\n", - "Factor = Primary (Mul / Div)*\n", - "Mul = \"*\" Primary\n", - "Div = \"/\" Primary\n", - "Primary = Parens / Neg / Number \n", - "Parens = \"(\" Term \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ~\"[0-9]+\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - "\n", - "AST = g.parse(\"2+3\") \n", - "eval(to_str(AST))" - ] - }, - { - "cell_type": "markdown", - "id": "68751235", - "metadata": {}, - "source": [ - "###Simplify\n", - "\n", - "Now, we don't care about the difference between addition and subtraction, or between multiplication and division, as we're going to treat them both the same, so lets simplify the grammar to take care of this case" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "52fc05c9", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "14" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "\n", - "Factor = Primary Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") Primary\n", - "\n", - "Primary = Parens / Neg / Number \n", - "Parens = \"(\" Term \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ~\"[0-9]+\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "g.parse(\"2+3\")\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - "\n", - "eval(to_str(g.parse(\"2+3*4\")))" - ] - }, - { - "cell_type": "markdown", - "id": "4283dc6e", - "metadata": {}, - "source": [ - "### Add floating point numbers\n", - "Now we'll go with a more complex number definition to try and capture floats" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "d396103b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "-1.6800000000000002" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "Factor = Primary Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") Primary\n", - "Primary = Parens / Neg / Number \n", - "Parens = \"(\" Term \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "g.parse(\"2+3\")\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - "\n", - "eval(to_str(g.parse(\"2.1+3*-4.2*.3\")))" - ] - }, - { - "cell_type": "markdown", - "id": "fd694257", - "metadata": {}, - "source": [ - "### Identifiers\n", - "\n", - "If we want to include variables in the schema, we need to be able to handle identifiers. Lets practice with an empty grammar to get it right." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "83f8c731", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "4" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "Keyword = 'int'\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - "\n", - "hi=4 \n", - "eval(to_str(g.parse(\"hi\")))" - ] - }, - { - "cell_type": "markdown", - "id": "01e6a0bf", - "metadata": {}, - "source": [ - "Now lets add the identifiers to the arithmetic we were working on previously" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "f83c13c2", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "28.400000000000002" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "Factor = Primary Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") Primary\n", - "Primary = Parens / Neg / Number / Identifier\n", - "Parens = \"(\" Term \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "Keyword = 'int' / 'exp'\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - "\n", - " \n", - "hi=4 \n", - "eval(to_str(g.parse(\"(5+hi)*3.1+.5\")))" - ] - }, - { - "cell_type": "markdown", - "id": "a0f79955", - "metadata": {}, - "source": [ - "### Add function calls\n", - "\n", - "Function calls are a primary unit in the order of operations. We explicitly spell out the keywords that are allowed to be used as function calls. If anything else comes in, it will throw an error. For starters, lets just use a few functions that we know python can handle, so we don't have to worry about translation." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "46d28297", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "-2.3245038118424985" - ] - }, - "execution_count": 7, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "Factor = Primary Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") Primary\n", - "Primary = Call / Parens / Neg / Number / Identifier\n", - "Parens = \"(\" Term \")\"\n", - "Call = Keyword \"(\" Term (\",\" Term)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "Keyword = 'exp' / 'sin' / 'cos'\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "def to_str(node):\n", - " if node.children:\n", - " return ''.join([to_str(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "hi=4 \n", - "eval(to_str(g.parse(\"cos(5+hi)*3.1+.5\")))" - ] - }, - { - "cell_type": "markdown", - "id": "72fa3f9b", - "metadata": {}, - "source": [ - "### Add exponentiation\n", - "Exponentiation adds another layer to our order of operations, and happens at the smallest unit, just above that of the primary elements. We put them in increasing order of priority, or from largest equation unit to smallest.\n", - "\n", - "As the python syntax for exponentiation is `**` instead of the SMILE standard `^`, we have to make our first translation. We do this by making a special case in the translation function which knows specifically what to do with an exponentive node when it sees one." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "id": "468b531c", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "9.3053961907966638" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier\n", - "Parens = \"(\" Term \")\"\n", - "Call = Keyword \"(\" Term (\",\" Term)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "\n", - "Keyword = 'exp' / 'sin' / 'cos'\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': #special case for translating exponent syntax\n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "hi=4 \n", - "eval(translate(g.parse(\"cos(sin(5+hi))*3.1^2+.5\")))\n", - "#eval(translate(g.parse(\"3+cos(5+hi)*3.1^2+.5\")))\n", - "#translate(g.parse(\"cos(5+hi)*3.1^2+.5\"))" - ] - }, - { - "cell_type": "markdown", - "id": "c0733eb1", - "metadata": {}, - "source": [ - "### Add translation of keywords\n", - "\n", - "As the names of functions in XMILE does not always match the names of functions in python, we'll add a dictionary to translate them, and a special case in the translation function that handles keyword nodes." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "id": "1a3b2dc3", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "61.12136331904043" - ] - }, - "execution_count": 9, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "#try translating keywords\n", - "#its important to get the keywords in the right order, so that 'exp' and 'exprnd' don't get confused.\n", - "\n", - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier\n", - "Parens = \"(\" Term \")\"\n", - "Call = Keyword \"(\" Term (\",\" Term)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "\n", - "Keyword = 'exprnd' / 'exp' / 'sin' / 'cos'\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "dictionary = {'exp':'exp', 'sin':'sin', 'cos':'cos', 'exprnd':'exponential'}\n", - "\n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': \n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\": # special case for translating keywords\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "hi=4 \n", - "eval(translate(g.parse(\"exprnd(5+hi)*3.1^2+.5\")))\n", - "#translate(g.parse(\"cos(5+hi)*3.1^2+.5\"))" - ] - }, - { - "cell_type": "markdown", - "id": "eb0f67d6", - "metadata": {}, - "source": [ - "###Add XMILE keywords\n", - "\n", - "Now that this structure is in place, lets add a bunch more keywords" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "id": "e0a6d2d4", - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "-8.2559618167117463" - ] - }, - "execution_count": 10, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# expand the keywords to include a goodly XMILE subset\n", - "grammar = \"\"\"\n", - "Term = Factor Additive*\n", - "Additive= (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier\n", - "Parens = \"(\" Term \")\"\n", - "Call = Keyword \"(\" Term (\",\" Term)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': # special case syntax change...\n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\":\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "hi = 4 \n", - "eval(translate(g.parse(\"cos(min(hi,6)+5)*3.1^2+.5\"))) " - ] - }, - { - "cell_type": "markdown", - "id": "17f14f07", - "metadata": {}, - "source": [ - "### Conditional behavior\n", - "\n", - "One of the xmile functions expects a boolean parameter, and so we had better add the ability to deal with conditional statements. These are even broader than addition and subtraction, and happen last in the order of operations - so naturally, the are first in our grammar." - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "id": "c24007d7", - "metadata": {}, - "outputs": [ - { - "ename": "ParseError", - "evalue": "Rule 'Condition' didn't match at 'absolutely_nothing' (line 1, column 1).", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[0;31mParseError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[1;32m 60\u001b[0m \u001b[0;31m#eval(translate(g.parse(\"cos(sin(5)+hi)*3.1^2+.5\")))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 61\u001b[0m \u001b[0;31m#translate(g.parse(\"cos(5+hi)*3.1^2+.5\"))\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 62\u001b[0;31m \u001b[0mtranslate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"absolutely_nothing\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m/Library/Python/2.7/site-packages/parsimonious/grammar.pyc\u001b[0m in \u001b[0;36mparse\u001b[0;34m(self, text, pos)\u001b[0m\n\u001b[1;32m 81\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 82\u001b[0m \u001b[0;34m\"\"\"Parse some text with the default rule.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdefault_rule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparse\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 85\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/Library/Python/2.7/site-packages/parsimonious/expressions.pyc\u001b[0m in \u001b[0;36mparse\u001b[0;34m(self, text, pos)\u001b[0m\n\u001b[1;32m 38\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 39\u001b[0m \"\"\"\n\u001b[0;32m---> 40\u001b[0;31m \u001b[0mnode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mpos\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 41\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m \u001b[0;34m<\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 42\u001b[0m \u001b[0;32mraise\u001b[0m \u001b[0mIncompleteParseError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mend\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;32m/Library/Python/2.7/site-packages/parsimonious/expressions.pyc\u001b[0m in \u001b[0;36mmatch\u001b[0;34m(self, text, pos)\u001b[0m\n\u001b[1;32m 55\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_match\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpos\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 56\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0merror\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m 58\u001b[0m \u001b[0;32mreturn\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m 59\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n", - "\u001b[0;31mParseError\u001b[0m: Rule 'Condition' didn't match at 'absolutely_nothing' (line 1, column 1)." - ] - } - ], - "source": [ - "grammar = \"\"\"\n", - "Condition = Term Conditional*\n", - "Conditional = (\"<=\" / \"<\" / \">=\" / \">\" / \"=\") Term\n", - "\n", - "Term = Factor Additive*\n", - "Additive = (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier \n", - "Parens = \"(\" Condition \")\"\n", - "Call = Keyword \"(\" Condition (\",\" Condition)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = !Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"* \n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': # special case syntax change...\n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\":\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "hi=4 \n", - "eval(translate(g.parse(\"exprnd(if_then_else(5>6,4,3)+hi)*3.1^2+.5\")))\n", - "#eval(translate(g.parse(\"if_then_else(5>6,4,3)\")))\n", - "#eval(translate(g.parse(\"int(5<=6)\")))\n", - "#eval(translate(g.parse(\"5<=6\")))\n", - "#eval(translate(g.parse(\"if_then_else(6,4,3)\")))\n", - "#eval(translate(g.parse(\"cos(min(5,hi,7)+5)*3.1^2+.5\")))\n", - "#eval(translate(g.parse(\"cos(sin(5)+hi)*3.1^2+.5\")))\n", - "#translate(g.parse(\"cos(5+hi)*3.1^2+.5\"))\n", - "translate(g.parse(\"absolutely_nothing\"))" - ] - }, - { - "cell_type": "markdown", - "id": "5bc351fe", - "metadata": {}, - "source": [ - "### Deal with identifiers that start with keywords\n", - "\n", - "If we give the previous method a test case like \"absolutely_nothing\" - something intended to be an identifier - it tries to parse it with the keyword, and then gets stuck\n", - "\n", - "One way to deal with this is to say that it is either something that is not a keyword, or its a keyword followed by at least one other character. \n", - "\n", - " Identifier = (!Keyword ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*) / (Keyword ~\"[a-z0-9_\\$]\"+)\n", - "\n", - "This is also problematic, as the tree builds up with a keyword in it, and that keyword gets replaced.\n", - "\n", - "Better to just make it a simple terminator:\n", - "\n", - " Identifier = ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*\n", - " \n", - "and count on the fact that we give precendence to keywords in the primary statement:\n", - "\n", - " Primary = Call / Parens / Neg / Number / Identifier " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25aef18b", - "metadata": {}, - "outputs": [], - "source": [ - "grammar = \"\"\"\n", - "Condition = Term Conditional*\n", - "Conditional = (\"<=\" / \"<\" / \">=\" / \">\" / \"=\") Term\n", - "\n", - "Term = Factor Additive*\n", - "Additive = (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier \n", - "Parens = \"(\" Condition \")\"\n", - "Call = Keyword \"(\" Condition (\",\" Condition)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*\n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': # special case syntax change...\n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\":\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "\n", - "translate(g.parse(\"absolutely_nothing\"))\n", - "translate(g.parse(\"normal_delivery_delay_recognized\"))" - ] - }, - { - "cell_type": "markdown", - "id": "873dd4de", - "metadata": {}, - "source": [ - "### return a list of dependancies\n", - "\n", - "List is a list of the identifiers present in the equation." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "53e9d7a5", - "metadata": {}, - "outputs": [], - "source": [ - "grammar = \"\"\"\n", - "Condition = Term Conditional*\n", - "Conditional = (\"<=\" / \"<\" / \">=\" / \">\" / \"=\") Term\n", - "\n", - "Term = Factor Additive*\n", - "Additive = (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier \n", - "Parens = \"(\" Condition \")\"\n", - "Call = Keyword \"(\" Condition (\",\" Condition)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*\n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def get_identifiers(node):\n", - " identifiers = []\n", - " for child in node:\n", - " for item in get_identifiers(child): #merge all into one list\n", - " identifiers.append(item)\n", - " if node.expr_name == 'Identifier':\n", - " identifiers.append(node.text)\n", - " return identifiers\n", - " \n", - "def translate(node):\n", - " if node.expr_name == 'Exponentive': # special case syntax change...\n", - " return '**' + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\":\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "\n", - " a = get_identifiers(g.parse(\"Robert*Mary+Cora+(Edith*Sybil)^Tom+int(Matthew)*Violet\".lower()))\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dcbe489b", - "metadata": {}, - "outputs": [], - "source": [ - "grammar = \"\"\"\n", - "Condition = Term Conditional*\n", - "Conditional = (\"<=\" / \"<\" / \">=\" / \">\" / \"=\") Term\n", - "\n", - "Term = Factor Additive*\n", - "Additive = (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier \n", - "Parens = \"(\" Condition \")\"\n", - "Call = Keyword \"(\" Condition (\",\" Condition)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*\n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\",\n", - " \"=\":\"==\", \"<=\":\"<=\", \"<\":\"<\", \">=\":\">=\", \">\":\">\", \"^\":\"**\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def get_identifiers(node):\n", - " identifiers = []\n", - " for child in node:\n", - " for item in get_identifiers(child): #merge all into one list\n", - " identifiers.append(item)\n", - " if node.expr_name == 'Identifier':\n", - " identifiers.append(node.text)\n", - " return identifiers\n", - " \n", - "def translate(node):\n", - " if node.expr_name in ['Exponentive', 'Conditional']: #non-terminal lookup\n", - " return dictionary[node.children[0].text] + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\": #terminal lookup\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "\n", - "translate(g.parse(\"2+3=4+5\"))\n", - "#print g.parse(\"2+3=4+5\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "70b4791e", - "metadata": {}, - "outputs": [], - "source": [ - "grammar = \"\"\"\n", - "Condition = Term Conditional*\n", - "Conditional = (\"<=\" / \"<\" / \">=\" / \">\" / \"=\") Term\n", - "\n", - "Term = Factor Additive*\n", - "Additive = (\"+\"/\"-\") Factor\n", - "\n", - "Factor = ExpBase Multiplicative*\n", - "Multiplicative = (\"*\" / \"/\") ExpBase\n", - "\n", - "ExpBase = Primary Exponentive*\n", - "Exponentive = \"^\" Primary\n", - "\n", - "Primary = Call / Parens / Neg / Number / Identifier \n", - "Parens = \"(\" Condition \")\"\n", - "Call = Keyword \"(\" Condition (\",\" Condition)* \")\"\n", - "Neg = \"-\" Primary\n", - "Number = ((~\"[0-9]\"+ \".\"? ~\"[0-9]\"*) / (\".\" ~\"[0-9]\"+)) ((\"e\"/\"E\") (\"-\"/\"+\") ~\"[0-9]\"+)?\n", - "Identifier = ~\"[a-z]\" ~\"[a-z0-9_\\$]\"*\n", - "\n", - "Keyword = \"exprnd\" / \"exp\" / \"sin\" / \"cos\" / \"abs\" / \"int\" / \"inf\" / \"log10\" / \"pi\" /\n", - " \"sqrt\" / \"tan\" / \"lognormal\" / \"normal\" / \"poisson\" / \"ln\" / \"min\" / \"max\" /\n", - " \"random\" / \"arccos\" / \"arcsin\" / \"arctan\" / \"if_then_else\"\n", - "\"\"\"\n", - "g = parsimonious.Grammar(grammar)\n", - "\n", - "\n", - "dictionary = {\"abs\":\"abs\", \"int\":\"int\", \"exp\":\"np.exp\", \"inf\":\"np.inf\", \"log10\":\"np.log10\",\n", - " \"pi\":\"np.pi\", \"sin\":\"np.sin\", \"cos\":\"np.cos\", \"sqrt\":\"np.sqrt\", \"tan\":\"np.tan\",\n", - " \"lognormal\":\"np.random.lognormal\", \"normal\":\"np.random.normal\", \n", - " \"poisson\":\"np.random.poisson\", \"ln\":\"np.ln\", \"exprnd\":\"np.random.exponential\",\n", - " \"random\":\"np.random.rand\", \"min\":\"min\", \"max\":\"max\", \"arccos\":\"np.arccos\",\n", - " \"arcsin\":\"np.arcsin\", \"arctan\":\"np.arctan\", \"if_then_else\":\"if_then_else\",\n", - " \"=\":\"==\", \"<=\":\"<=\", \"<\":\"<\", \">=\":\">=\", \">\":\">\", \"^\":\"**\"}\n", - "\n", - "#provide a few functions\n", - "def if_then_else(condition, val_if_true, val_if_false):\n", - " if condition:\n", - " return val_if_true\n", - " else:\n", - " return val_if_false\n", - "\n", - "def get_identifiers(node):\n", - "# identifiers = []\n", - "# for child in node:\n", - "# for item in get_identifiers(child): #merge all into one list\n", - "# identifiers.append(item)\n", - "# if node.expr_name == 'Identifier':\n", - "# identifiers.append(node.text)\n", - "# return identifiers\n", - " identifiers = []\n", - " for child in node:\n", - " identifiers += get_identifiers(child)\n", - " identifiers += [node.text] if node.expr_name in ['Identifier'] else []\n", - " return identifiers\n", - " \n", - "def translate(node):\n", - " if node.expr_name in ['Exponentive', 'Conditional']: #non-terminal lookup\n", - " return dictionary[node.children[0].text] + ''.join([translate(child) for child in node.children[1:]])\n", - " elif node.expr_name == \"Keyword\": #terminal lookup\n", - " return dictionary[node.text]\n", - " else:\n", - " if node.children:\n", - " return ''.join([translate(child) for child in node])\n", - " else:\n", - " return node.text\n", - " \n", - "a = get_identifiers(g.parse(\"Robert*Mary+Cora+(Edith*Sybil)^Tom+int(Matthew)*Violet\".lower()))\n", - "print a\n", - "#translate(g.parse(\"2+3=4+5\"))\n", - "#print g.parse(\"2+3=4+5\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.7" - } - }, - "nbformat": 4, - "nbformat_minor": 1 -} diff --git a/docs/development/SMILEv4.pdf b/docs/development/SMILEv4.pdf deleted file mode 100644 index 44ec6794..00000000 Binary files a/docs/development/SMILEv4.pdf and /dev/null differ diff --git a/docs/development/XMILEv4.pdf b/docs/development/XMILEv4.pdf deleted file mode 100644 index 8def7c5f..00000000 Binary files a/docs/development/XMILEv4.pdf and /dev/null differ diff --git a/docs/development/development_index.rst b/docs/development/development_index.rst index 1c61ff15..133ce3bc 100644 --- a/docs/development/development_index.rst +++ b/docs/development/development_index.rst @@ -1,13 +1,14 @@ Developer Documentation ======================= - .. toctree:: - :maxdepth: 2 + :hidden: - about - pysd_architecture_views/4+1view_model - contributing + guidelines pathway - structure - complement + pysd_architecture_views/4+1view_model + +In order to contribut to PySD check the :doc:`guidelines` and the :doc:`pathway`. +You also will find helpful the :doc:`Structure of the PySD library <../../structure/structure_index>` to understand better how it works. + + diff --git a/docs/development/contributing.rst b/docs/development/guidelines.rst similarity index 85% rename from docs/development/contributing.rst rename to docs/development/guidelines.rst index ba6ba04c..b792e4f5 100644 --- a/docs/development/contributing.rst +++ b/docs/development/guidelines.rst @@ -1,15 +1,15 @@ -Contributing to PySD -==================== +Development Guidelines +====================== If you are interested in helping to develop PySD, the :doc:`pathway` lists areas that are ripe for contribution. To get started, you can fork the repository and make contributions to your own version. -When you're happy with your edits, submit a pull request to the main branch. +When you are happy with your edits, submit a pull request to the main branch. .. note:: - In order to open a pull request,the new features and changes should be througly tested. + In order to open a pull request, the new features and changes should be througly tested. To do so, unit tests of new features or translated functions should be added, please check the Development Tools section below. When opening a pull request all tests are run and the coverage and pep8 style are checked. Development Tools @@ -40,7 +40,7 @@ complementary tests in the corresponding `unit_test_*.py` file. Speed Tests ^^^^^^^^^^^ -The speed tests may be developed in the future. Any contribution is welcome. +Speed tests may be developed in the future. Any contribution is welcome. Profiler @@ -54,17 +54,17 @@ The profiler depends on :py:mod:`cProfile` and `cprofilev `_ is a module that checks that your code meets proper python +`Pylint `_ is a module that checks that your code meets proper Python coding practices. It is helpful for making sure that the code will be easy for other people to read, and also is good fast feedback for improving your coding practice. The lint checker can be run for -the entire packages, and for individual python modules or classes. It should be run at a local level +the entire packages, and for individual Python modules or classes. It should be run at a local level (ie, on specific files) whenever changes are made, and globally before the package is committed. It doesn't need to be perfect, but we should aspire always to move in a positive direction.' PySD Design Philosophy ---------------------- -Understanding that a focussed project is both more robust and maintainable, PySD aspires to the +Understanding that a focussed project is both more robust and maintainable, PySD adheres to the following philosophy: @@ -73,15 +73,15 @@ following philosophy: * Anything that is not endemic to System Dynamics (such as plotting, integration, fitting, etc) should either be implemented using external tools, or omitted. * Stick to SD. Let other disciplines (ABM, Discrete Event Simulation, etc) create their own tools. - * Use external model creation tools + * Use external model creation tools. -* Use the core language of system dynamics. +* Use the core language of System Dynamics. * Limit implementation to the basic XMILE standard. * Resist the urge to include everything that shows up in all vendors' tools. -* Emphasize ease of use. Let SD practitioners who haven't used python before understand the basics. -* Take advantage of general python constructions and best practices. +* Emphasize ease of use. Let SD practitioners who haven't used Python before understand the basics. +* Take advantage of general Python constructions and best practices. * Develop and use strong testing and profiling components. Share your work early. Find bugs early. * Avoid firefighting or rushing to add features quickly. SD knows enough about short term thinking in software development to know where that path leads. diff --git a/docs/development/internal_functions.rst b/docs/development/internal_functions.rst deleted file mode 100644 index 22b4da1f..00000000 --- a/docs/development/internal_functions.rst +++ /dev/null @@ -1,52 +0,0 @@ -Internal Functions -================== - -This section documents the functions that are going on behaind the scenes, for the benefit of developers. - -Special functions needed for model execution --------------------------------------------- - -.. automodule:: pysd.py_backend.components - :members: - :undoc-members: - :private-members: - -.. automodule:: pysd.py_backend.statefuls - :members: - :undoc-members: - :private-members: - -.. automodule:: pysd.py_backend.functions - :members: - :undoc-members: - :private-members: - -.. automodule:: pysd.py_backend.utils - :members: - :undoc-members: - :private-members: - -Building the python model file ------------------------------- - -.. automodule:: pysd.translation.builder - :members: - :undoc-members: - :private-members: - - -External data reading ---------------------- - -.. automodule:: pysd.py_backend.external - :members: - :undoc-members: - :private-members: - - -Decorators used in the model file ---------------------------------- -.. automodule:: pysd.py_backend.decorators - :members: - :undoc-members: - :private-members: \ No newline at end of file diff --git a/docs/development/pathway.rst b/docs/development/pathway.rst index cedbe8db..bbefeeb4 100644 --- a/docs/development/pathway.rst +++ b/docs/development/pathway.rst @@ -2,7 +2,7 @@ PySD Development Pathway ======================== High priority features, bugs, and other elements of active effort are listed on the `github issue -tracker. `_ To get involved see :doc:`contributing`. +tracker. `_ To get involved see :doc:`guidelines`. High Priority @@ -14,7 +14,7 @@ High Priority Medium Priority --------------- -* Improve model exexution speed using cython, theano, numba, or another package +* Improve model execution speed using cython, theano, numba, or another package Low Priority @@ -42,8 +42,8 @@ Current Features * Basic XMILE and Vensim parser * Established library structure and data formats -* Simulation using existing python integration tools -* Integration with basic python Data Science functionality +* Simulation using existing Python integration tools +* Integration with basic Python Data Science functionality * Run-at-a-time parameter modification * Time-variant exogenous inputs * Extended backends for storing parameters and output values diff --git a/docs/development/pysd_architecture_views/4+1view_model.rst b/docs/development/pysd_architecture_views/4+1view_model.rst index 3470fee6..8e8105f5 100644 --- a/docs/development/pysd_architecture_views/4+1view_model.rst +++ b/docs/development/pysd_architecture_views/4+1view_model.rst @@ -2,6 +2,11 @@ The "4+1" Model View of Software Architecture ============================================= .. _4+1 model view: https://www.cs.ubc.ca/~gregor/teaching/papers/4+1view-architecture.pdf + +.. warning:: + This page is outdated as it was written for PySD 2.x. However, the content here could be useful for developers. + For PySD 3+ architecture see :doc:`Structure of the PySD module <../../structure/structure_index>`. + The `4+1 model view`_, designed by Philippe Krutchen, presents a way to describe the architecture of software systems, using multiple and concurrent views. This use of multiple views allows to address separately the concerns of the various 'stakeholders' of the architecture such as end-user, developers, systems engineers, project managers, etc. The software architecture deals with abstraction, with decomposition and composition, with style and system's esthetic. To describe a software architecture, we use a model formed by multiple views or perspectives. That model is made up of five main views: logical view, development view, process view, physical view and scenarios or user cases. diff --git a/docs/development/structure.rst b/docs/development/structure.rst deleted file mode 100644 index 546abb59..00000000 --- a/docs/development/structure.rst +++ /dev/null @@ -1,61 +0,0 @@ -Structure of the PySD module -============================ - -PySD provides a set of translators that interpret a Vensim or XMILE format model into a Python native class. The model components object represents the state of the system, and contains methods that compute auxiliary and flow variables based upon the current state. - -The components object is wrapped within a Python class that provides methods for modifying and executing the model. These three pieces constitute the core functionality of the PySD module, and allow it to interact with the Python data analytics stack. - - - -Translation -^^^^^^^^^^^ - -The internal functions of the model translation components can be seen in the following documents - -.. toctree:: - :maxdepth: 2 - - vensim_translation - xmile_translation - - -The PySD module is capable of importing models from a Vensim model file (\*.mdl) or an XMILE format xml file. Translation makes use of a Parsing Expression Grammar parser, using the third party Python library Parsimonious13 to construct an abstract syntax tree based upon the full model file (in the case of Vensim) or individual expressions (in the case of XMILE). - -The translators then crawl the tree, using a dictionary to translate Vensim or Xmile syntax into its appropriate Python equivalent. The use of a translation dictionary for all syntactic and programmatic components prevents execution of arbitrary code from unverified model files, and ensures that we only translate commands that PySD is equipped to handle. Any unsupported model functionality should therefore be discovered at import, instead of at runtime. - -The use of a one-to-one dictionary in translation means that the breadth of functionality is inherently limited. In the case where no direct Python equivalent is available, PySD provides a library of functions such as pulse, step, etc. that are specific to dynamic model behavior. - -In addition to translating individual commands between Vensim/XMILE and Python, PySD reworks component identifiers to be Python-safe by replacing spaces with underscores. The translator allows source identifiers to make use of alphanumeric characters, spaces, or the $ symbol. - -During translation some dictionaries are created that allow the correct operation of the model: - -* **_namespace**: used to connect real name (from the original model) with the Python name. -* **_subscript_dict**: Used to define the subscript range and subranges. -* **_dependencies**: Used to define the dependencies of each variable and assign cache type and initialize the model. - - -The model class -^^^^^^^^^^^^^^^ -The translator constructs a Python class that represents the system dynamics model. The class maintains a dictionary representing the current values of each of the system stocks, and the current simulation time, making it a ‘statefull’ model in much the same way that the system itself has a specific state at any point in time. - -The model class also contains a function for each of the model components, representing the essential model equations. The docstring for each function contains the model documentation and units as translated from the original model file. A query to any of the model functions will calculate and return its value according to the stored state of the system. - -The model class maintains only a single state of the system in memory, meaning that all functions must obey the Markov property - that the future state of the system can be calculated entirely based upon its current state. In addition to simplifying integration, this requirement enables analyses that interact with the model at a step-by-step level. The downside to this design choice is that several components of Vensim or XMILE functionality – the most significant being the infinite order delay – are intentionally not supported. In many cases similar behavior can be approximated through other constructs. - -Lastly, the model class provides a set of methods that are used to facilitate simulation. PySD uses the standard ordinary differential equations solver provided in the well-established Python library Scipy, which expects the state and its derivative to be represented as an ordered list. The model class provides the function .d_dt() that takes a state vector from the integrator and uses it to update the model state, and then calculates the derivative of each stock, returning them in a corresponding vector. A complementary function .state_vector() creates an ordered vector of states for use in initializing the integrator. - -The PySD class -^^^^^^^^^^^^^^ -.. toctree:: - :maxdepth: 2 - - internal_functions - - - - -The PySD class provides the machinery to get the model moving, supply it with data, or modify its parameters. In addition, this class is the primary way that users interact with the PySD module. - -The basic function for executing a model is appropriately named.run(). This function passes the model into scipy’s odeint() ordinary differential equations solver. The scipy integrator is itself utilizing the lsoda integrator from the Fortran library odepack14, and so integration takes advantage of highly optimized low-level routines to improve speed. We use the model’s timestep to set the maximum step size for the integrator’s adaptive solver to ensure that the integrator properly accounts for discontinuities. - -The .run() function returns to the user a Pandas dataframe representing the output of their simulation run. A variety of options allow the user to specify which components of the model they would like returned, and the timestamps at which they would like those measurements. Additional parameters make parameter changes to the model, modify its starting conditions, or specify how simulation results should be logged. \ No newline at end of file diff --git a/docs/development/supported_vensim_functions.rst b/docs/development/supported_vensim_functions.rst deleted file mode 100644 index 087d218a..00000000 --- a/docs/development/supported_vensim_functions.rst +++ /dev/null @@ -1,121 +0,0 @@ -+------------------------------+------------------------------+ -| Vensim | Python Translation | -+==============================+==============================+ -| = | == | -+------------------------------+------------------------------+ -| < | < | -+------------------------------+------------------------------+ -| > | > | -+------------------------------+------------------------------+ -| >= | >= | -+------------------------------+------------------------------+ -| <= | <= | -+------------------------------+------------------------------+ -| ^ | \** | -+------------------------------+------------------------------+ -| ABS | np.abs | -+------------------------------+------------------------------+ -| MIN | np.minimum | -+------------------------------+------------------------------+ -| MAX | np.maximum | -+------------------------------+------------------------------+ -| SQRT | np.sqrt | -+------------------------------+------------------------------+ -| EXP | np.exp | -+------------------------------+------------------------------+ -| LN | np.log | -+------------------------------+------------------------------+ -| PI | np.pi | -+------------------------------+------------------------------+ -| SIN | np.sin | -+------------------------------+------------------------------+ -| COS | np.cos | -+------------------------------+------------------------------+ -| TAN | np.tan | -+------------------------------+------------------------------+ -| ARCSIN | np.arcsin | -+------------------------------+------------------------------+ -| ARCCOS | np.arccos | -+------------------------------+------------------------------+ -| ARCTAN | np.arctan | -+------------------------------+------------------------------+ -| ELMCOUNT | len | -+------------------------------+------------------------------+ -| INTEGER | functions.integer | -+------------------------------+------------------------------+ -| QUANTUM | functions.quantum | -+------------------------------+------------------------------+ -| MODULO | functions.modulo | -+------------------------------+------------------------------+ -| IF THEN ELSE | functions.if_then_else | -+------------------------------+------------------------------+ -| PULSE TRAIN | functions.pulse_train | -+------------------------------+------------------------------+ -| RAMP | functions.ramp | -+------------------------------+------------------------------+ -| INVERT MATRIX | functions.invert_matrix | -+------------------------------+------------------------------+ -| VMIN | functions.vmin | -+------------------------------+------------------------------+ -| VMAX | functions.vmax | -+------------------------------+------------------------------+ -| SUM | functions.sum | -+------------------------------+------------------------------+ -| PROD | functions.prod | -+------------------------------+------------------------------+ -| LOGNORMAL | np.random.lognormal | -+------------------------------+------------------------------+ -| STEP | functions.step | -+------------------------------+------------------------------+ -| PULSE | functions.pulse | -+------------------------------+------------------------------+ -| EXPRND | np.random.exponential | -+------------------------------+------------------------------+ -| POISSON | np.random.poisson | -+------------------------------+------------------------------+ -| RANDOM NORMAL | functions.bounded_normal | -+------------------------------+------------------------------+ -| RANDOM UNIFORM | np.random.rand | -+------------------------------+------------------------------+ -| DELAY1 | functions.Delay | -+------------------------------+------------------------------+ -| DELAY3 | functions.Delay | -+------------------------------+------------------------------+ -| DELAY N | functions.DelayN | -+------------------------------+------------------------------+ -| DELAY FIXED | functions.DelayFixed | -+------------------------------+------------------------------+ -| FORECAST | functions.Forecast | -+------------------------------+------------------------------+ -| SAMPLE IF TRUE | functions.SampleIfTrue | -+------------------------------+------------------------------+ -| SMOOTH3 | functions.Smooth | -+------------------------------+------------------------------+ -| SMOOTH N | functions.Smooth | -+------------------------------+------------------------------+ -| SMOOTH | functions.Smooth | -+------------------------------+------------------------------+ -| INITIAL | functions.Initial | -+------------------------------+------------------------------+ -| XIDZ | functions.XIDZ | -+------------------------------+------------------------------+ -| ZIDZ | functions.XIDZ | -+------------------------------+------------------------------+ -| GET XLS DATA | external.ExtData | -+------------------------------+------------------------------+ -| GET DIRECT DATA | external.ExtData | -+------------------------------+------------------------------+ -| GET XLS LOOKUPS | external.ExtLookup | -+------------------------------+------------------------------+ -| GET DIRECT LOOKUPS | external.ExtLookup | -+------------------------------+------------------------------+ -| GET XLS CONSTANTS | external.ExtConstant | -+------------------------------+------------------------------+ -| GET DIRECT CONSTANTS | external.ExtConstant | -+------------------------------+------------------------------+ -| GET XLS SUBSCRIPT | external.ExtSubscript | -+------------------------------+------------------------------+ -| GET DIRECT SUBSCRIPT | external.ExtSubscript | -+------------------------------+------------------------------+ - - `np` corresponds to the numpy package diff --git a/docs/development/vensim_translation.rst b/docs/development/vensim_translation.rst deleted file mode 100644 index ecc1727b..00000000 --- a/docs/development/vensim_translation.rst +++ /dev/null @@ -1,41 +0,0 @@ -Vensim Translation -================== - -PySD parses a vensim '.mdl' file and translates the result into python, creating a new file in the -same directory as the original. For example, the Vensim file `Teacup.mdl `_ becomes `Teacup.py `_ . - -This allows model execution independent of the Vensim environment, which can be handy for deploying -models as backends to other products, or for performing massively parallel distributed computation. - -These translated model files are read by PySD, which provides methods for modifying or running the -model and conveniently accessing simulation results. - - -Translated Functions --------------------- - -Ongoing development of the translator will support the full subset of Vensim functionality that -has an equivalent in XMILE. The current release supports the following functionality: - -.. include:: supported_vensim_functions.rst - -Additionally, identifiers are currently limited to alphanumeric characters and the dollar sign $. - -Future releases will include support for: - -- subscripts -- arrays -- arbitrary identifiers - -There are some constructs (such as tagging variables as 'suplementary') which are not currently -parsed, and may throw an error. Future releases will handle this with more grace. - - -Used Functions for Translation ------------------------------- - -.. automodule:: pysd.translation.vensim.vensim2py - :members: - :undoc-members: - :private-members: - diff --git a/docs/development/xmile_translation.rst b/docs/development/xmile_translation.rst deleted file mode 100644 index e4b96314..00000000 --- a/docs/development/xmile_translation.rst +++ /dev/null @@ -1,7 +0,0 @@ -XMILE Translation -================= - -The XMILE reference documentation is located at: - -* XMILE: http://www.iseesystems.com/community/support/XMILEv4.pdf -* SMILE: http://www.iseesystems.com/community/support/SMILEv4.pdf \ No newline at end of file diff --git a/docs/functions.rst b/docs/functions.rst deleted file mode 100644 index 1a844796..00000000 --- a/docs/functions.rst +++ /dev/null @@ -1,12 +0,0 @@ -User Functions Reference -======================== - -These are the primary functions that control model import and execution. - - -.. autofunction:: pysd.read_vensim - -.. autofunction:: pysd.read_xmile - -.. autofunction:: pysd.load - diff --git a/docs/generate_tables.py b/docs/generate_tables.py new file mode 100644 index 00000000..29fa20ca --- /dev/null +++ b/docs/generate_tables.py @@ -0,0 +1,66 @@ +import pandas as pd +from pathlib import Path + + +def generate(table, columns, output): + """Generate markdown table.""" + # select only the given columns + subtable = table[columns] + # remove the rows where the first column is na + subtable = subtable[~subtable[columns[0]].isna()] + + if all(subtable[columns[-1]].isna()): + # if the commnets columns (last) is all na, do not save it + subtable = subtable[columns[:-1]] + + # Place an empty string where na values + subtable.values[subtable.isna()] = "" + + if len(subtable.index) > 1: + # Save values only if the table has rows + print(f"Table generated: {output}") + subtable.to_csv(output, index=None) + + +def generate_tables(): + """Generate markdown tables for documentation.""" + + tables_dir = Path(__file__).parent / "tables" + + # different tables to load + tables = { + "binary": tables_dir / "binary.tab", + "unary": tables_dir / "unary.tab", + "functions": tables_dir / "functions.tab", + "delay_functions": tables_dir / "delay_functions.tab", + "get_functions": tables_dir / "get_functions.tab" + } + + # different combinations to generate + contents = { + "vensim": [ + "Vensim", "Vensim example", "Abstract Syntax", "Vensim comments" + ], + "xmile": [ + "Xmile", "Xmile example", "Abstract Syntax", "Xmile comments" + ], + "python": [ + "Abstract Syntax", "Python Translation", "Python comments" + ] + } + + # load the tables + tables = {key: pd.read_table(value) for key, value in tables.items()} + + # generate the tables + for table, df in tables.items(): + for language, content in contents.items(): + generate( + df, + content, + tables_dir / f"{table}_{language}.csv" + ) + + # transform arithmetic order table + file = tables_dir / "arithmetic.tab" + pd.read_table(file).to_csv(file.with_suffix(".csv"), index=None) diff --git a/docs/getting_started.rst b/docs/getting_started.rst new file mode 100644 index 00000000..d3cf8456 --- /dev/null +++ b/docs/getting_started.rst @@ -0,0 +1,248 @@ +Getting Started +=============== + +Importing a model and getting started +------------------------------------- +To begin, we must first load the PySD module, and use it to import a model file:: + + >>> import pysd + >>> model = pysd.read_vensim('Teacup.mdl') + + +This code creates an instance of the :doc:`PySD Model class ` from an example model that we will use as the system dynamics equivalent of ‘Hello World’: a cup of tea cooling at room temperature. + +.. image:: images/Teacup.png + :width: 350 px + :align: center + +.. note:: + The teacup model can be found in the `samples of the test-models repository `_. + +To view a synopsis of the model equations and documentation, use the :py:attr:`.doc` property of the Model class. This will generate a listing of all model elements, their documentation, units, and initial values, where appropriate, and return them as a :py:class:`pandas.DataFrame`. Here is a sample from the teacup model:: + + >>> model.doc + + Real Name Py Name Subscripts Units Limits Type Subtype Comment + 0 Characteristic Time characteristic_time None Minutes (0.0, nan) Constant Normal How long will it take the teacup to cool 1/e o... + 1 FINAL TIME final_time None Minute (nan, nan) Constant Normal The final time for the simulation. + 2 Heat Loss to Room heat_loss_to_room None Degrees Fahrenheit/Minute (nan, nan) Auxiliary Normal This is the rate at which heat flows from the ... + 3 INITIAL TIME initial_time None Minute (nan, nan) Constant Normal The initial time for the simulation. + 4 Room Temperature room_temperature None Degrees Fahrenheit (-459.67, nan) Constant Normal Put in a check to ensure the room temperature ... + 5 SAVEPER saveper None Minute (0.0, nan) Auxiliary Normal The frequency with which output is stored. + 6 TIME STEP time_step None Minute (0.0, nan) Constant Normal The time step for the simulation. + 7 Teacup Temperature teacup_temperature None Degrees Fahrenheit (32.0, 212.0) Stateful Integ The model is only valid for the liquid phase o... + 8 Time time None None (nan, nan) None None Current time of the model. + + +.. note:: + You can also load an already translated model file. This will be faster than loading an original model, as the translation is not required:: + + >>> import pysd + >>> model = pysd.load('Teacup.py') + +.. note:: + The functions :py:func:`pysd.read_vensim()`, :py:func:`pysd.read_xmile()` and :py:func:`pysd.load()` have optional arguments for advanced usage. You can check the full description in :doc:`Model loading ` or using :py:func:`help()` e.g.:: + + >>> import pysd + >>> help(pysd.load) + +.. note:: + Not all the features and functions are implemented. If you are in trouble while importing a Vensim or Xmile model check the :ref:`Vensim supported functions ` or :ref:`Xmile supported functions `. + +Running the Model +----------------- +The simplest way to simulate the model is to use the :py:meth:`.run` command with no options. This runs the model with the default parameters supplied in the model file, and returns a :py:class:`pandas.DataFrame` of the values of the model components at every timestamp:: + + >>> stocks = model.run() + >>> stocks + + Characteristic Time Heat Loss to Room Room Temperature Teacup Temperature FINAL TIME INITIAL TIME SAVEPER TIME STEP + 0.000 10 11.000000 70 180.000000 30 0 0.125 0.125 + 0.125 10 10.862500 70 178.625000 30 0 0.125 0.125 + 0.250 10 10.726719 70 177.267188 30 0 0.125 0.125 + 0.375 10 10.592635 70 175.926348 30 0 0.125 0.125 + 0.500 10 10.460227 70 174.602268 30 0 0.125 0.125 + ... ... ... ... ... ... ... ... ... + 29.500 10 0.565131 70 75.651312 30 0 0.125 0.125 + 29.625 10 0.558067 70 75.580671 30 0 0.125 0.125 + 29.750 10 0.551091 70 75.510912 30 0 0.125 0.125 + 29.875 10 0.544203 70 75.442026 30 0 0.125 0.125 + 30.000 10 0.537400 70 75.374001 30 0 0.125 0.125 + +[241 rows x 8 columns] + +Pandas proovides a simple plotting capability, that we can use to see how the temperature of the teacup evolves over time:: + + >>> import matplotlib.pyplot as plt + >>> stocks["Teacup Temperature"].plot() + >>> plt.title("Teacup Temperature") + >>> plt.ylabel("Degrees F") + >>> plt.xlabel("Minutes") + >>> plt.grid() + +.. image:: images/Teacup_Cooling.png + :width: 400 px + :align: center + +To show a progressbar during the model integration, the `progress` argument can be passed to the :py:meth:`.run` method:: + + >>> stocks = model.run(progress=True) + +.. note:: + The full description of the :py:meth:`.run` method and other methods can be found in the :doc:`Model methods section `. + +Running models with DATA type components +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Venim allows to import DATA type data from binary `.vdf` files. Variables defined without an equation in the model, will attempt to read their values from the `.vdf`. PySD allows running models with this kind of data definition using the data_files argument when calling :py:meth:`.run` command, e.g.:: + + >>> stocks = model.run(data_files="input_data.tab") + +Several files can be passed by using a list. If the data information is not found in the first file, the next one will be used until finding the data values:: + + >>> stocks = model.run(data_files=["input_data.tab", "input_data2.tab", ..., "input_datan.tab"]) + +If a variables are defined in different files, to choose the specific file a dictionary can be used:: + + >>> stocks = model.run(data_files={"input_data.tab": ["data_var1", "data_var3"], "input_data2.tab": ["data_var2"]}) + +.. note:: + Only `tab` and `csv` files are supported. They should be given as a table, with each variable in a column (or row) and the time in the first column (or first row). The column (or row) names can be given using the name of the variable in the original model or using python names. + +.. note:: + Subscripted variables must be given in the Vensim format, one column (or row) per subscript combination. Example of column names for 2x2 variable: + `subs var[A, C]` `subs var[B, C]` `subs var[A, D]` `subs var[B, D]` + +Outputting various run information +---------------------------------- +The :py:meth:`.run` command has a few options that make it more useful. In many situations we want to access components of the model other than merely the stocks - we can specify which components of the model should be included in the returned dataframe by including them in a list that we pass to the :py:meth:`.run` command, using the return_columns keyword argument:: + + >>> model.run(return_columns=['Teacup Temperature', 'Room Temperature']) + + Teacup Temperature Room Temperature + 0.000 180.000000 70 + 0.125 178.625000 70 + 0.250 177.267188 70 + 0.375 175.926348 70 + 0.500 174.602268 70 + ... ... ... + 29.500 75.651312 70 + 29.625 75.580671 70 + 29.750 75.510912 70 + 29.875 75.442026 70 + 30.000 75.374001 70 + + [241 rows x 2 columns] + + +If the measured data that we are comparing with our model comes in at irregular timestamps, we may want to sample the model at timestamps to match. The :py:meth:`.run` function provides this functionality with the return_timestamps keyword argument:: + + >>> model.run(return_timestamps=[0, 1, 3, 7, 9.5, 13, 21, 25, 30]) + + Characteristic Time Heat Loss to Room Room Temperature Teacup Temperature FINAL TIME INITIAL TIME SAVEPER TIME STEP + 0.0 10 11.000000 70 180.000000 30 0 0.125 0.125 + 1.0 10 9.946940 70 169.469405 30 0 0.125 0.125 + 3.0 10 8.133607 70 151.336071 30 0 0.125 0.125 + 7.0 10 5.438392 70 124.383922 30 0 0.125 0.125 + 9.5 10 4.228756 70 112.287559 30 0 0.125 0.125 + 13.0 10 2.973388 70 99.733876 30 0 0.125 0.125 + 21.0 10 1.329310 70 83.293098 30 0 0.125 0.125 + 25.0 10 0.888819 70 78.888194 30 0 0.125 0.125 + 30.0 10 0.537400 70 75.374001 30 0 0.125 0.125 + + +Retrieving a flat DataFrame +--------------------------- +The subscripted variables, in general, will be returned as :py:class:`xarray.DataArray` in the output :py:class:`pandas.DataFrame`. To get a flat dataframe, set `flatten=True` when calling the :py:meth:`.run` method:: + + >>> model.run(flatten=True) + +Setting parameter values +------------------------ +In some situations we may want to modify the parameters of the model to investigate its behavior under different assumptions. There are several ways to do this in PySD, but the :py:meth:`.run` method gives us a convenient method in the `params` keyword argument. + +This argument expects a dictionary whose keys correspond to the components of the model. The associated values can either be constants, or :py:class:`pandas.Series` whose indices are timestamps and whose values are the values that the model component should take on at the corresponding time. For instance, in our model we may set the room temperature to a constant value:: + + >>> model.run(params={'Room Temperature': 20}) + +Alternately, if we want the room temperature to vary over the course of the simulation, we can give the :py:meth:`.run` method a set of time-series values in the form of a :py:class:`pandas.Series`, and PySD will linearly interpolate between the given values in the course of its integration:: + + >>> import pandas as pd + >>> temp = pd.Series(index=range(30), data=range(20, 80, 2)) + >>> model.run(params={'Room Temperature': temp}) + +If the parameter value to change is a subscripted variable (vector, matrix...), there are three different options to set the new value. Suposse we have ‘Subscripted var’ with dims :py:data:`['dim1', 'dim2']` and coordinates :py:data:`{'dim1': [1, 2], 'dim2': [1, 2]}`. A constant value can be used and all the values will be replaced:: + + >>> model.run(params={'Subscripted var': 0}) + +A partial :py:class:`xarray.DataArray` can be used. For example a new variable with ‘dim2’ but not ‘dim2’. In that case, the result will be repeated in the remaining dimensions:: + + >>> import xarray as xr + >>> new_value = xr.DataArray([1, 5], {'dim2': [1, 2]}, ['dim2']) + >>> model.run(params={'Subscripted var': new_value}) + +Same dimensions :py:class:`xarray.DataArray` can be used (recommended):: + + >>> import xarray as xr + >>> new_value = xr.DataArray([[1, 5], [3, 4]], {'dim1': [1, 2], 'dim2': [1, 2]}, ['dim1', 'dim2']) + >>> model.run(params={'Subscripted var': new_value}) + +In the same way, a :py:class:`pandas.Series` can be used with constant values, partially defined :py:class:`xarray.DataArray` or same dimensions :py:class:`xarray.DataArray`. + +.. note:: + Once parameters are set by the :py:meth:`.run` command, they are permanently changed within the model. We can also change model parameters without running the model, using PySD’s :py:meth:`.set_components` method, which takes the same params dictionary as the :py:meth:`.run` method. We might choose to do this in situations where we will be running the model many times, and only want to set the parameters once. + +.. note:: + If you need to know the dimensions of a variable, you can check them by using :py:meth:`.get_coords` method:: + + >>> model.get_coords('Room Temperature') + + None + + >>> model.get_coords('Subscripted var') + + ({'dim1': [1, 2], 'dim2': [1, 2]}, ['dim1', 'dim2']) + + this will return the coords dictionary and the dimensions list, if the variable is subscripted, or ‘None’ if the variable is an scalar. + +.. note:: + If you change the value of a lookup function by a constant, the constant value will be used always. If a :py:class:`pandas.Series` is given the index and values will be used for interpolation when the function is called in the model, keeping the arguments that are included in the model file. + + If you change the value of any other variable type by a constant, the constant value will be used always. If a :py:class:`pandas.Series` is given the index and values will be used for interpolation when the function is called in the model, using the time as argument. + + If you need to know if a variable takes arguments, i.e., if it is a lookup variable, you can check it by using the :py:meth:`.get_args` method:: + + >>> model.get_args('Room Temperature') + + [] + + >>> model.get_args('Growth lookup') + + ['x'] + +Setting simulation initial conditions +------------------------------------- +Initial conditions for our model can be set in several ways. So far, we have used the default value for the `initial_condition` keyword argument, which is ‘original’. This value runs the model from the initial conditions that were specified originally in the model file. We can alternately specify a tuple containing the start time and a dictionary of values for the system's stocks. Here we start the model with the tea at just above freezing temperature:: + + >>> model.run(initial_condition=(0, {'Teacup Temperature': 33})) + +The new value can be a :py:class:`xarray.DataArray`, as explained in the previous section. + +Additionally, we can run the model forward from its current position, by passing initial_condition=‘current’. After having run the model from time zero to thirty, we can ask the model to continue running forward for another chunk of time:: + + >>> model.run(initial_condition='current', + return_timestamps=range(31, 45)) + +The integration picks up at the last value returned in the previous run condition, and returns values at the requested timestamps. + +There are times when we may choose to overwrite a stock with a constant value (ie, for testing). To do this, we just use the params value, as before. Be careful not to use 'params' when you really mean to be setting the initial condition! + + +Querying current values +----------------------- +We can easily access the current value of a model component using curly brackets. For instance, to find the temperature of the teacup, we simply call:: + + >>> model['Teacup Temperature'] + +If you try to get the current values of a lookup variable, the previous method will fail, as lookup variables take arguments. However, it is possible to get the full series of a lookup or data object with :py:meth:`.get_series_data` method:: + + >>> model.get_series_data('Growth lookup') diff --git a/docs/images/Vensim_file.svg b/docs/images/Vensim_file.svg new file mode 100644 index 00000000..8bef89c1 --- /dev/null +++ b/docs/images/Vensim_file.svg @@ -0,0 +1,1387 @@ + + + +FilecontentSketchSection(main)Section(macro)Element(my value)SubscriptRange(dim)Component +(my value[A])Component +(my value[B])Component +(my value[C]){UTF-8}:MACRO: transform(input, parameter)transform = input/intermediate~Dmnl~Transformation of the input.|intermediate = input - parameter~Units~Difference respecte the parameter.|:END OF MACRO:dim: A, B, C ~~|inflow = Time~Month [0,?]~Inflow|myvalue[A] = transform(stock, 1) ~~|my value[B] = transform(stock, 2) ~~|my value[C] = transform(stock, 3)~Dmnl~First, second and third order transformations of stock.|outflow = SQRT(Time)~Month [0,?]~Outflow.|stock = INTEG (inflow-outflow, 0)~Month~Stock.|********************************************************.Control********************************************************~Simulation Control Parameters|FINAL TIME = 10~Month~The final time for the simulation.|INITIAL TIME = 0~Month~The initial time for the simulation.|SAVEPER = TIME STEP~Month [0,?]~The frequency with which output is stored.|TIME STEP = 1~Month [0,?]~The time step for the simulation.|\\\---/// Sketch information - do not modify anything except namesV300 Do not put anything below this section - it will be ignored*View 1$192-192-192,0,Times New Roman|12||0-0-0|0-0-0|0-0-255|-1--1--1|-1--1--1|96,96,100,010,1,stock,566,259,40,20,3,3,0,0,0,0,0,012,2,48,810,261,10,8,0,3,0,0,-1,0,0,01,3,5,2,4,0,0,22,0,0,0,-1--1--1,,1|(754,261)|1,4,5,1,100,0,0,22,0,0,0,-1--1--1,,1|(651,261)|11,5,48,703,261,6,8,34,3,0,0,1,0,0,010,6,outflow,703,280,25,11,40,3,0,0,-1,0,0,012,7,48,330,264,10,8,0,3,0,0,-1,0,0,01,8,10,1,4,0,0,22,0,0,0,-1--1--1,,1|(482,264)|1,9,10,7,100,0,0,22,0,0,0,-1--1--1,,1|(383,264)|11,10,48,433,264,6,8,34,3,0,0,1,0,0,010,11,inflow,433,283,20,11,40,3,0,0,-1,0,0,010,12,Time,573,356,26,11,8,2,0,3,-1,0,0,0,128-128-128,0-0-0,|12||128-128-1281,13,12,11,0,0,0,0,0,64,0,-1--1--1,,1|(508,322)|1,14,12,6,0,0,0,0,0,128,0,-1--1--1,,1|(631,321)|10,15,my value,569,142,29,11,8,3,0,0,0,0,0,01,16,1,15,0,0,0,0,0,128,0,-1--1--1,,1|(566,202)|///---\\\UnitsLimitsDocumentation diff --git a/docs/images/abstract_model.png b/docs/images/abstract_model.png new file mode 100644 index 00000000..6302193e Binary files /dev/null and b/docs/images/abstract_model.png differ diff --git a/docs/index.rst b/docs/index.rst index 1e4b1c21..99d7d54d 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -34,29 +34,21 @@ PySD .. |DOI| image:: https://zenodo.org/badge/DOI/10.5281/zenodo.5654824.svg :target: https://doi.org/10.5281/zenodo.5654824 -This project is a simple library for running System Dynamics models in python, with the purpose of -improving integration of Big Data and Machine Learning into the SD workflow. +This project is a simple library for running System Dynamics models in Python, with the purpose of improving integration of Big Data and Machine Learning into the SD workflow. -PySD translates :doc:`Vensim ` or -:doc:`XMILE ` model files into python modules, -and provides methods to modify, simulate, and observe those translated models. +PySD translates :doc:`Vensim ` or +:doc:`XMILE ` model files into Python modules, +and provides methods to modify, simulate, and observe those translated models. The translation is done throught an intermediate :doc:`Abstract Synatax Tree representation `, +which makes it possible to add builders in other languages in a simpler way +Why create a new SD simulation engine? +-------------------------------------- -Contents: ---------- +There are a number of great SD programs out there (`Vensim `_, `iThink `_, `AnyLogic `_, `Insight Maker `_, and `others `_). In order not to waste our effort, or fall victim to the `Not-Invented-Here `_ fallacy, we should have a very good reason for starting a new project. -.. toctree:: - :maxdepth: 2 - - installation - basic_usage - advanced_usage - command_line_usage - tools - functions - development/development_index - reporting_bugs +That reason is this: There is a whole world of computational tools being developed in the larger data science community. **System dynamicists should directly use the tools that other people are building, instead of replicating their functionality in SD specific software.** The best way to do this is to bring specific SD functionality to the domain where those other tools are being developed. +This approach allows SD modelers to take advantage of the most recent developments in data science, and focus our efforts on improving the part of the stack that is unique to System Dynamics modeling. Additional Resources -------------------- @@ -66,7 +58,7 @@ PySD Cookbook A cookbook of simple recipes for advanced data analytics using PySD is available at: http://pysd-cookbook.readthedocs.org/ -The cookbook includes models, sample data, and code in the form of ipython notebooks that demonstrate a variety of data integration and analysis tasks. These models can be executed on your local machine, and modified to suit your particular analysis requirements. +The cookbook includes models, sample data, and code in the form of iPython notebooks that demonstrate a variety of data integration and analysis tasks. These models can be executed on your local machine, and modified to suit your particular analysis requirements. Contributing @@ -92,3 +84,20 @@ You can also cite the library using the `DOI provided by Zenodo `: +In order to plot model outputs as shown in :doc:`Getting started <../getting_started>`: * Matplotlib @@ -77,7 +76,7 @@ These Python libraries bring additional data analytics capabilities to the analy Additionally, the System Dynamics Translator utility developed by Robert Ward is useful for translating models from other system dynamics formats into the XMILE standard, to be read by PySD. -These modules can be installed using pip with syntax similar to the above. +These modules can be installed using pip with a syntax similar to the above. Additional Resources diff --git a/docs/python_api/functions.rst b/docs/python_api/functions.rst new file mode 100644 index 00000000..04a25a3f --- /dev/null +++ b/docs/python_api/functions.rst @@ -0,0 +1,11 @@ +Python functions and stateful objects +===================================== +Functions +--------- +.. automodule:: pysd.py_backend.functions + :members: + +Statefuls +--------- +.. automodule:: pysd.py_backend.statefuls + :members: \ No newline at end of file diff --git a/docs/python_api/model_class.rst b/docs/python_api/model_class.rst new file mode 100644 index 00000000..55f918d3 --- /dev/null +++ b/docs/python_api/model_class.rst @@ -0,0 +1,13 @@ +Python model class +================== + +Model class +----------- +.. autoclass:: pysd.py_backend.model.Model + :members: + + +Macro class +----------- +.. autoclass:: pysd.py_backend.model.Macro + :members: diff --git a/docs/python_api/model_loading.rst b/docs/python_api/model_loading.rst new file mode 100644 index 00000000..51d55bff --- /dev/null +++ b/docs/python_api/model_loading.rst @@ -0,0 +1,11 @@ +Model loading +============= +For loading a translated model with Python the function :py:func:`pysd.load` can be used: + +.. autofunction:: pysd.load + +To translate a load a model the :py:func:`pysd.read_vensim` and :py:func:`pysd.read_xmile` functions can be used: + +.. autofunction:: pysd.read_vensim + +.. autofunction:: pysd.read_xmile \ No newline at end of file diff --git a/docs/python_api/python_api_index.rst b/docs/python_api/python_api_index.rst new file mode 100644 index 00000000..0b5d9d7e --- /dev/null +++ b/docs/python_api/python_api_index.rst @@ -0,0 +1,40 @@ +Python API +========== + +.. toctree:: + :hidden: + + model_loading + model_class + functions + +This sections describes the main functions and functionalities to translate +models to Python and run them. If you need more detailed description about +the translation and building process, please see the :doc:`../structure/structure_index` section. + +The model loading information can be found in :doc:`model_loading` and consists of the following functions: + +.. list-table:: Translating and loading functions + :widths: 25 75 + :header-rows: 0 + + * - :py:func:`pysd.read_vensim` + - Translates a Vensim file to Python and returns a :py:class:`Model` object. + * - :py:func:`pysd.read_xmile` + - Translates a Xmile file to Python and returns a :py:class:`Model` object. + * - :py:func:`pysd.load` + - Loads a transtaled Python file and returns a :py:class:`Model` object. + +The Model and Macro classes information ad public methods and attributes can be found in :doc:`model_class`. + +.. list-table:: Translating and loading functions + :widths: 25 75 + :header-rows: 0 + + * - :py:class:`pysd.py_backend.model.Model` + - Implements functionalities to load a translated model and interact with it. The :py:class:`Model` class inherits from :py:class:`Macro`, therefore, some public methods and properties are defined in the :py:class:`Macro` class. + * - :py:class:`pysd.py_backend.model.Macro` + - Implements functionalities to load a translated macro and interact with it. Most of its core methods are also use by :py:class:`Model` class. + + +Provided functions and stateful classes to integrate python models are described in :doc:`functions`. diff --git a/docs/reporting_bugs.rst b/docs/reporting_bugs.rst index 6a24fb16..860a4c67 100644 --- a/docs/reporting_bugs.rst +++ b/docs/reporting_bugs.rst @@ -1,21 +1,24 @@ Reporting bugs ============== -Before reporting any bug, please make sure that you are using the latest version of PySD, you can get your version by running `python -m pysd -v` on the command line. +Before reporting any bug, please make sure that you are using the latest version of PySD. You can get the version number by running `python -m pysd -v` on the command line. -All the bugs must be reported in the project's `issue tracker on github `_. +All bugs must be reported in the project's `issue tracker on github `_. + +.. note:: + Not all the features and functions are implemented. If you are in trouble while translating or running a Vensim or Xmile model check the :ref:`Vensim supported functions ` or :ref:`Xmile supported functions ` and consider that when openning a new issue. Bugs during translation ----------------------- 1. Check the line where it happened and try to identify if it is due to a missing function or feature or for any other reason. -2. See if there is any opened issue with the same of similar bug. If it is you can add there your specific problem. -3. If there is no similar issue, open a new one. Try to use a descriptive title such us `Missing subscripts support for Xmile models`, avoid titles like `Error when parsing Xmile model`. Provide the given error information, and if possible, a small model reproducing the same error. +2. See if there is any open issue with the same or a similar bug. If there is, you can add your specific problem there. +3. If not previously reported, open a new issue. Try to use a descriptive title such us `Missing subscripts support for Xmile models`, avoid titles like `Error while parsing Xmile model`. Provide the given error information and, if possible, a small model reproducing the same error. Bugs during runtime ------------------- -1. Check if similar bug has been detected on the issue tracker. If not open a new issue with a descriptive title. +1. Check if a similar bug has been reported on the issue tracker. If that is not the case, open a new issue with a descriptive title. 2. Provide the error information and all the relevant lines you used to execute the model. -3. If possible provide a small model reproducing the bug. +3. If possible, provide a small model reproducing the bug. diff --git a/docs/structure/abstract_model.rst b/docs/structure/abstract_model.rst new file mode 100644 index 00000000..2e117fb3 --- /dev/null +++ b/docs/structure/abstract_model.rst @@ -0,0 +1,31 @@ +Abstract Model Representation +============================= +The Abstract Model representation allows a separation of concern between +translation and building. Translation involves anything that +happens from the moment the source code of the original model is loaded +into memory up to the creation of the Abstract Model representation. Similarly, +the building will be everything that takes place between the Abstract Model and the +source code of the model written in a programming language different than that +of the original model.This approach allows to easily include new code to the translation or or building process, +without the the risk of affecting one another. + +The :py:class:`AbstractModel` object should retain as much information from the +original model as possible. Although the information is not used +in the output code, it may be necessary for other future output languages +or for improvements in the currently supported outputs. For example, currently +unchangeable constansts (== defined in Vensim) are treated as regular +components with Python, but in the future we may want to protect them +from user interaction. + +The lowest level of this representation is the :py:class:`AbstractSyntax` Tree (AST). +This includes all the operations and calls in a given component expression. + +Main abstract structures +------------------------ +.. automodule:: pysd.translators.structures.abstract_model + :members: + +Abstrat structures for the AST +------------------------------ +.. automodule:: pysd.translators.structures.abstract_expressions + :members: diff --git a/docs/structure/python_builder.rst b/docs/structure/python_builder.rst new file mode 100644 index 00000000..5c8d5eb5 --- /dev/null +++ b/docs/structure/python_builder.rst @@ -0,0 +1,67 @@ +Python builder +============== + +The Python builder allows to build models that can be run with the PySD Model class. + +The use of a one-to-one dictionary in translation means that the breadth of functionality is inherently limited. In the case where no direct Python equivalent is available, PySD provides a library of functions such as `pulse`, `step`, etc. that are specific to dynamic model behavior. + +In addition to translating individual commands between Vensim/XMILE and Python, PySD reworks component identifiers to be Python-safe by replacing spaces with underscores. The translator allows source identifiers to make use of alphanumeric characters, spaces, or special characteres. In order to make that possible a namespace is created, which links the original name of the variable with the Python-safe name. The namespace is also available in the PySD model class to allow users working with both original names and Python-safe names. + + +Main builders +------------- +.. automodule:: pysd.builders.python.python_model_builder + :members: + +Expression builders +------------------- +.. automodule:: pysd.builders.python.python_expressions_builder + :members: + + +.. _Python supported functions: + +Supported expressions examples +------------------------------ +Operators +^^^^^^^^^ + +.. csv-table:: Supported unary operators + :file: ../tables/unary_python.csv + :header-rows: 1 + +.. csv-table:: Supported binary operators + :file: ../tables/binary_python.csv + :header-rows: 1 + +Functions +^^^^^^^^^ + +.. csv-table:: Supported basic functions + :file: ../tables/functions_python.csv + :header-rows: 1 + +.. csv-table:: Supported delay functions + :file: ../tables/delay_functions_python.csv + :header-rows: 1 + +.. csv-table:: Supported get functions + :file: ../tables/get_functions_python.csv + :header-rows: 1 + +Namespace manager +----------------- +.. automodule:: pysd.builders.python.namespace + :members: NamespaceManager + + +Subscript manager +----------------- +.. automodule:: pysd.builders.python.subscripts + :members: + + +Imports manager +--------------- +.. automodule:: pysd.builders.python.imports + :members: diff --git a/docs/structure/structure_index.rst b/docs/structure/structure_index.rst new file mode 100644 index 00000000..e981f832 --- /dev/null +++ b/docs/structure/structure_index.rst @@ -0,0 +1,71 @@ +Structure of the PySD library +============================= + +PySD provides translators that allow to convert the original model into an Abstract Model Representation (AMR), or :doc:`Abstract Model ` for short. This representation allows to gather all the model equations and behavior into a number of Python data classes. Therefore, the AMR is Python code, hence independent of the programming language used to write the original model. The AMR is then passed to a builder, which converts it to source code of a programming language of our choice. See the example of the complete process in the figure below. + +.. image:: ../images/abstract_model.png + :width: 700 px + :align: center + +Currently, PySD can translate Vensim models (mdl format) or models in Xmile format (exported from Vensim, Stella or other software) into an AMR. The only builder available at the moment builds the models in Python. + +For models translated to Python, all the necessary functions and classes to run them are included in PySD. The :py:class:`Model` class is the main class that allows loading and running a model, as well as modifying the values of its parameters, among many other possibilities. + +Translation +----------- + +.. toctree:: + :hidden: + + vensim_translation + xmile_translation + abstract_model + +PySD currentlty supports translation :doc:`from Vensim ` amb :doc:`from Xmile `. + +PySD can import models in Vensim's \*.mdl file format and in XMILE format (\*.xml, \*.xmile, or \*.stmx file). `Parsimonious `_ is the Parsing Expression Grammar `(PEG) `_ parser library used in PySD to parse the original models and construct an abstract syntax tree. The translators then crawl the tree, using a set of classes to define the :doc:`Abstract Model `. + +When parsing the expressions of any language, the order of operations must be taken into account. The order is shown in the following table and is used to create :py:class:`ArithmeticStructure` and :py:class:`LogicalStructure` objects correctly. The following expression :py:data:`1+2*3-5`` will be translated to:: + + ArithmeticStructure(operators=['+', '-'], arguments=(1, ArithmeticStructure(operators=['*'], arguments=(2, 3)), 5)) + +While something like :py:data:`1<5 and 5>3`:: + + LogicStructure(operators=[':AND:'], arguments=(LogicStructure(operators=['<'], arguments=(1, 5)), LogicStructure(operators=['>'], arguments=(5, 3)))) + +The parenthesis also affects same order operatos, for example :py:data:`1+2-3` is translated to:: + + ArithmeticStructure(operators=['+', '-'], arguments=(1, 2, 3)) + +While :py:data:`1+(2-3)` is translated to:: + + ArithmeticStructure(operators=['+'], arguments=(1, ArithmeticStructure(operators=['-'], arguments=(2, 3)))) + +It is important to maintain this order because although these operations by definition are commutative due to the numerical error due to the precision, they may not be commutative in the integration. + +.. csv-table:: Arithmetic order + :file: ../tables/arithmetic.csv + :header-rows: 1 + + +Building the model +------------------ + +.. toctree:: + :hidden: + + python_builder + +The builders allow to build the final model in any programming language (so long as there is a builder for that particular language). To do so, they use a series of classes that obtain the information from the :doc:`Abstract Model ` and convert it into the desired code. Currently PySD only includes a :doc:`builder to build the models in Python ` . Any contribution to add new builders (and solvers) for other programming languages is welcome. + + +The Python model +---------------- + +For loading a translated model with Python see :doc:`Getting started <../../getting_started>` or :doc:`Model loading <../../python_api/model_loading>`. The Python builder constructs a Python class that represents the system dynamics model. The class maintains a dictionary representing the current values of each of the system stocks, and the current simulation time, making it a `stateful` model in much the same way that the system itself has a specific state at any point in time. + +The :doc:`Model class <../../python_api/model_class>` also contains a function for each of the model components, representing the essential model equations. Each function contains its units, subcscripts type infromation and documentation as translated from the original model file. A query to any of the model functions will calculate and return its value according to the stored state of the system. + +The :doc:`Model class <../../python_api/model_class>` maintains only a single state of the system in memory, meaning that all functions must obey the Markov property - that the future state of the system can be calculated entirely based upon its current state. In addition to simplifying integration, this requirement enables analyses that interact with the model at a step-by-step level. + +Lastly, the :doc:`Model class <../../python_api/model_class>` provides a set of methods that are used to facilitate simulation. The :py:meth:`.run` method returns to the user a Pandas dataframe representing the output of their simulation run. A variety of options allow the user to specify which components of the model they would like returned, and the timestamps at which they would like those measurements. Additional parameters make parameter changes to the model, modify its starting conditions, or specify how simulation results should be logged. diff --git a/docs/structure/vensim_translation.rst b/docs/structure/vensim_translation.rst new file mode 100644 index 00000000..d1796459 --- /dev/null +++ b/docs/structure/vensim_translation.rst @@ -0,0 +1,121 @@ +Vensim Translation +================== + +PySD allows parsing a Vensim `.mdl` file and translates the result to an :py:class:`AbstractModel` object that can later (building process) be used to build the model in another programming language. + + +Translation workflow +------------------------- +The following translation workflow allows splitting the Vensim file while parsing its contents in order to build an :py:class:`AbstractModel` object. The workflow may be summarized as follows: + +1. **Vensim file**: splits the model equations from the sketch and allows splitting the model in sections (main section and macro sections). +2. **Vensim section**: is a full set of varibles and definitions that is integrable. The Vensim section can then be split into model expressions. +3. **Vensim element**: a definition in the mdl file which could be a subscript (sub)range definition or a variable definition. It includes units and comments. Definitions for the same variable are grouped after in the same :py:class:`AbstractElement` object. Allows parsing its left hand side (LHS) to get the name of the subscript (sub)range or variable and it is returned as a specific type of component depending on the used assing operator (=, ==, :=, (), :) +4. **Vensim component**: the classified object for a variable definition, it depends on the opperator used to define the variable. Its right hand side (RHS) can be parsed to get the Abstract Syntax Tree (AST) of the expression. + +Once the model is parsed and broken following the previous steps, the :py:class:`AbstractModel` is returned. + + +.. image:: ../images/Vensim_file.svg + :alt: Vensim file parts + + +Vensim file +^^^^^^^^^^^ + +.. automodule:: pysd.translators.vensim.vensim_file + :members: VensimFile + :undoc-members: + +Vensim section +^^^^^^^^^^^^^^ + +.. automodule:: pysd.translators.vensim.vensim_section + :members: Section + :undoc-members: + +Vensim element +^^^^^^^^^^^^^^ + +.. automodule:: pysd.translators.vensim.vensim_element + :members: SubscriptRange, Element, Component, UnchangeableConstant, Data, Lookup + :undoc-members: + + +.. _Vensim supported functions: + +Supported Functions and Features +-------------------------------- + +Ongoing development of the translator will support the full subset of Vensim functionality. The current release supports the following operators, functions and features. + +Operators +^^^^^^^^^ +All the basic operators are supported, this includes the ones shown in the tables below. + +.. csv-table:: Supported unary operators + :file: ../tables/unary_vensim.csv + :header-rows: 1 + +.. csv-table:: Supported binary operators + :file: ../tables/binary_vensim.csv + :header-rows: 1 + +Moreover, the Vensim :EXCEPT: operator is also supported to manage exceptions in the subscripts. See the :ref:`Subscripts section` section. + +Functions +^^^^^^^^^ +The list of currentlty supported Vensim functions are detailed below: + +.. csv-table:: Supported basic functions + :file: ../tables/functions_vensim.csv + :header-rows: 1 + +.. csv-table:: Supported delay functions + :file: ../tables/delay_functions_vensim.csv + :header-rows: 1 + +.. csv-table:: Supported get functions + :file: ../tables/get_functions_vensim.csv + :header-rows: 1 + + +Stocks +^^^^^^ +Stocks defined in Vensim as `INTEG(flow, initial_value)` are supported and are translated to the AST as `IntegStructure(flow, initial_value)`. + + +.. _Subscripts section: + +Subscripts +^^^^^^^^^^ +Several subscript related features are also supported. Thiese include: + +- Basic subscript operations with different ranges. +- Subscript ranges and subranges definitions. +- Basic subscript mapping, where the subscript range is mapping to a full range (e.g. new_dim: A, B, C -> dim, dim_other). Mapping to a partial range is not yet supported (e.g. new_dim: A, B, C -> dim: E, F, G). +- Subscript copy (e.g. new_dim <-> dim). +- \:EXCEPT: operator with any number of arguments. +- Subscript usage as a variable (e.g. my_var[dim] = another var * dim). +- Subscript vectorial opperations (e.g. SUM(my var[dim, dim!])). + +Lookups +^^^^^^^ +Vensim Lookups expressions are supported. They can be defined using hardcoded values, using `GET LOOKUPS` function or using `WITH LOOKUPS` function. + +Data +^^^^ +Data definitions with GET functions and empty data definitions (no expressions, Vensim uses a VDF file) are supported. These definitions may or may not include any of the possible interpolation keywords: :INTERPOLATE:, :LOOK FORWARD:, :HOLD BACKWARD:, :RAW:. These keywords will be stored in the 'keyword' argument of :py:class:`AbstractData` as 'interpolate', 'look_forward', 'hold_backward' and 'raw', respectively. The Abstract Structure for GET XLS/DATA is given in the supported GET functions table. The Abstract Structure for the empty Data declarations is a :py:class:`DataStructure`. + +For the moment, any specific functions applying over data are supported (e.g. SHIFT IF TRUE, TIME SHIFT...), but new ones may be includded in the future. + +Macro +^^^^^ +Vensim macros are supported. The macro content between the keywords \:MACRO: and \:END OF MACRO: is classified as a section of the model and is subsequently sused to build an independent section from the rest of the model. + +Planed New Functions and Features +--------------------------------- +- ALLOCATE BY PRIORITY +- GET TIME VALUE +- SHIFT IF TRUE +- VECTOR SELECT diff --git a/docs/structure/xmile_translation.rst b/docs/structure/xmile_translation.rst new file mode 100644 index 00000000..c8908706 --- /dev/null +++ b/docs/structure/xmile_translation.rst @@ -0,0 +1,106 @@ +Xmile Translation +================= + +PySD allows parsing a Xmile file and translates the result to an :py:class:`AbstractModel` object that can be used to builde the model. + + +.. warning:: + Currently no Xmile users are working on the development of PySD. This is causing a gap between the Xmile and Vensim developments. Stella users are encouraged to take part in the development of PySD by includying new `test models `_ and adding support for new functions and features. + + +The translation workflow +------------------------- +The following translation workflow allows splitting the Xmile file while parsing each part of it to build an :py:class:`AbstractModel` type object. The workflow may be summarized as follows: + +1. **Xmile file**: Parses the file with etree library and creates a section for the model. +2. **Xmile section**: Full set of varibles and definitions that can be integrated. Allows splitting the model elements. +3. **Xmile element**: A variable definition. It includes units and commnets. Allows parsing the expressions it contains and saving them inside AbstractComponents, that are part of an AbstractElement. + +Once the model is parsed and split following the previous steps. The :py:class:`AbstractModel` can be returned. + + +Xmile file +^^^^^^^^^^ + +.. automodule:: pysd.translators.xmile.xmile_file + :members: XmileFile + :undoc-members: + +Xmile section +^^^^^^^^^^^^^ + +.. automodule:: pysd.translators.xmile.xmile_section + :members: Section + :undoc-members: + +Xmile element +^^^^^^^^^^^^^ + +.. automodule:: pysd.translators.xmile.xmile_element + :members: SubscriptRange, Element, Flaux, Gf, Stock + :undoc-members: + + +.. _Xmile supported functions: + +Supported Functions and Features +-------------------------------- + +Ongoing development of the translator will support the full set of Xmile functionality. The current release supports the following operators, functions and features: + +.. warning:: + Not all the supported functions and features are properly tested. Any new test model to cover the missing functions test will be welcome. + +Operators +^^^^^^^^^ +All the basic operators are supported, this includes the ones shown in the tables below.: + +.. csv-table:: Supported unary operators + :file: ../tables/unary_xmile.csv + :header-rows: 1 + +.. csv-table:: Supported binary operators + :file: ../tables/binary_xmile.csv + :header-rows: 1 + + +Functions +^^^^^^^^^ +Not all the Xmile functions are included yet. The list of supported functions is shown below: + +.. csv-table:: Supported basic functions + :file: ../tables/functions_xmile.csv + :header-rows: 1 + +.. csv-table:: Supported delay functions + :file: ../tables/delay_functions_xmile.csv + :header-rows: 1 + + +Stocks +^^^^^^ +Stocks are supported with any number of inflows and outflows. Stocks are translated to the AST as `IntegStructure(flows, initial_value)`. + +Subscripts +^^^^^^^^^^ +Several subscript related features are supported. Thiese include: + +- Basic subscript operations with different ranges. +- Subscript ranges and subranges definitions. + +Graphical functions +^^^^^^^^^^^^^^^^^^^ +Xmile graphical functions (gf), also known as lookups, are supported. They can be hardcoded or inlined. + +.. warning:: + Interpolation methods 'extrapolate' and 'discrete' are implemented but not tested. Full integration models with these methods are required. + +Supported in Vensim but not in Xmile +------------------------------------ +Macro +^^^^^ +Currently Xmile macros are not supported. In Vensim, macros are classified as an independent section of the model. If they are properly parsed in the :py:class:`XmileFile`, adding support for Xmile should be easy. + +Planed New Functions and Features +--------------------------------- +Nothing yet. diff --git a/docs/tables/arithmetic.tab b/docs/tables/arithmetic.tab new file mode 100644 index 00000000..8e14c4ff --- /dev/null +++ b/docs/tables/arithmetic.tab @@ -0,0 +1,10 @@ +Arithmetic order Operators Operations +0 "(), None" "parenthesis, function call, references, no-operations" +1 "\-" negative value +2 ^ exponentation +3 "\*, /" "multiplication, division" +4 "%" modulo +5 "+, -" "addition, substraction" +6 "=, <>, <, <=, >, >=" comparison +7 "not" unary logical operation +8 "and, or" binary logical operations diff --git a/docs/tables/binary.tab b/docs/tables/binary.tab new file mode 100644 index 00000000..d7ec0c2c --- /dev/null +++ b/docs/tables/binary.tab @@ -0,0 +1,15 @@ +Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments +^ A ^ B ^ A ^ B "ArithmeticStructure(['^'], (A, B))" A**B +"\*" A * B "\*" A * B "ArithmeticStructure(['*'], (A, B))" A*B +/ A / B / A / B "ArithmeticStructure(['/'], (A, B))" A/B + mod A mod B "CallStructure('modulo', (A, B))" "pysd.functions.modulo(A, B)" +"\+" A + B "\+" A + B "ArithmeticStructure(['+'], (A, B))" A+B +"\-" A - B "\-" A - B "ArithmeticStructure(['-'], (A, B))" A-B += A = B = A = B "LogicStructure(['='], (A, B))" A == B +<> A <> B <> A <> B "LogicStructure(['<>'], (A, B))" A != B +< A < B < A < B "LogicStructure(['<'], (A, B))" A < B +> A > B > A > B "LogicStructure(['>'], (A, B))" A > B +>= A >= B >= A >= B "LogicStructure(['>='], (A, B))" A >= B +<= A <= B <= A <= B "LogicStructure(['<='], (A, B))" A <= B +"\:AND:" A :AND: B and A and B "LogicStructure([':AND:'], (A, B))" "numpy.and(A, B)" +"\:OR:" A :OR: B or A or B "LogicStructure([':OR:'], (A, B))" "numpy.or(A, B)" diff --git a/docs/tables/delay_functions.tab b/docs/tables/delay_functions.tab new file mode 100644 index 00000000..cd209660 --- /dev/null +++ b/docs/tables/delay_functions.tab @@ -0,0 +1,18 @@ +Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments +DELAY1I "DELAY1I(input, delay_time, initial_value)" delay1 "delay1(input, delay_time, initial_value)" "DelayStructure(input, delay_time, initial_value, 1)" pysd.statefuls.Delay(...) Not tested for Xmile! +DELAY1 "DELAY1(input, delay_time)" delay1 "delay1(input, delay_time)" "DelayStructure(input, delay_time, input, 1)" pysd.statefuls.Delay(...) Not tested for Xmile! +DELAY3I "DELAY3I(input, delay_time, initial_value)" delay3 "delay3(input, delay_time, initial_value)" "DelayStructure(input, delay_time, initial_value, 3)" pysd.statefuls.Delay(...) Not tested for Xmile! +DELAY3 "DELAY3(input, delay_time)" delay3 "delay3(input, delay_time)" "DelayStructure(input, delay_time, input, 3)" pysd.statefuls.Delay(...) Not tested for Xmile! +DELAY N "DELAY N(input, delay_time, initial_value, n)" delayn "delayn(input, delay_time, n, initial_value)" "DelayNStructure(input, delay_time, initial_value, n)" pysd.statefuls.DelayN(...) Not tested for Xmile! + delayn "delayn(input, delay_time, n)" "DelayNStructure(input, delay_time, input, n)" pysd.statefuls.DelayN(...) Not tested for Xmile! +DELAY FIXED "DELAY FIXED(input, delay_time, initial_value)" "DelayFixed(input, delay_time, initial_value)" pysd.statefuls.DelayFixed(...) Not tested for Xmile! +SMOOTHI "SMOOTH1I(input, delay_time, initial_value)" smth1 "smth1(input, smth_time, initial_value)" "SmoothStructure(input, smth_time, initial_value, 1)" pysd.statefuls.Smooth(...) Not tested for Xmile! +SMOOTH "SMOOTH1(input, delay_time)" smth1 "smth1(input, smth_time)" "SmoothStructure(input, smth_time, input, 1)" pysd.statefuls.Smooth(...) Not tested for Xmile! +SMOOTH3I "SMOOTH3I(input, delay_time, initial_value)" smth3 "smth3(input, smth_time, initial_value)" "SmoothStructure(input, smth_time, initial_value, 3)" pysd.statefuls.Smooth(...) Not tested for Xmile! +SMOOTH3 "SMOOTH3(input, delay_time)" smth3 "smth3(input, smth_time)" "SmoothStructure(input, smth_time, input, 3)" pysd.statefuls.Smooth(...) Not tested for Xmile! +SMOOTH N "SMOOTH N(input, delay_time, initial_value, n)" smthn "smthn(input, smth_time, n, initial_value)" "SmoothNStructure(input, smth_time, initial_value, n)" pysd.statefuls.SmoothN(...) Not tested for Xmile! + smthn "smthn(input, smth_time, n)" "SmoothNStructure(input, smth_time, input, n)" pysd.statefuls.SmoothN(...) Not tested for Xmile! + forcst "forcst(input, average_time, horizon, initial_trend)" "ForecastStructure(input, average_time, horizon, initial_trend)" pysd.statefuls.Forecast(...) Not tested for Xmile! +FORECAST "FORECAST(input, average_time, horizon)" forcst "forcst(input, average_time, horizon)" "ForecastStructure(input, average_time, horizon, 0)" pysd.statefuls.Forecast(...) Not tested for Xmile! +TREND "TREND(input, average_time, initial_trend)" trend "trend(input, average_time, initial_trend)" "TrendStructure(input, average_time, initial_trend)" pysd.statefuls.Trend(...) Not tested for Xmile! + trend "trend(input, average_time)" "TrendStructure(input, average_time, 0)" pysd.statefuls.Trend(...) Not tested for Xmile! diff --git a/docs/tables/functions.tab b/docs/tables/functions.tab new file mode 100644 index 00000000..2377741f --- /dev/null +++ b/docs/tables/functions.tab @@ -0,0 +1,38 @@ +Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments +ABS ABS(A) abs(A) abs(A) "CallStructure('abs', (A,))" numpy.abs(A) +MIN "MIN(A, B)" min "min(A, B)" "CallStructure('min', (A, B))" "numpy.minimum(A, B)" +MAX "MAX(A, B)" max "max(A, B)" "CallStructure('max', (A, B))" "numpy.maximum(A, B)" +SQRT SQRT(A) sqrt sqrt(A) "CallStructure('sqrt', (A,))" numpy.sqrt +EXP EXP(A) exp exp(A) "CallStructure('exp', (A,))" numpy.exp(A) +LN LN(A) ln ln(A) "CallStructure('ln', (A,))" numpy.log(A) +SIN SIN(A) sin sin(A) "CallStructure('sin', (A,))" numpy.sin(A) +COS COS(A) cos cos(A) "CallStructure('cos', (A,))" numpy.cos(A) +TAN TAN(A) tan tan(A) "CallStructure('tan', (A,))" numpy.tan(A) +ARCSIN ARCSIN(A) arcsin arcsin(A) "CallStructure('arcsin', (A,))" numpy.arcsin(A) +ARCCOS ARCCOS(A) arccos arccos(A) "CallStructure('arccos', (A,))" numpy.arccos(A) +ARCTAN ARCTAN(A) arctan arctan(A) "CallStructure('arctan', (A,))" numpy.arctan(A) +INVERT MATRIX INVERT MATRIX(A) "CallStructure('invert_matrix', (A,))" pysd.functions.invert_matrix(A) +ELMCOUNT ELMCOUNT(A) "CallStructure('elmcount', (A,))" len(A) +INTEGER INTEGER(A) int int(A) "CallStructure('int', (A,))" pysd.functions.integer(A) +QUANTUM "QUANTUM(A, B)" "CallStructure('quantum', (A, B))" "pysd.functions.quantum(A, B)" +MODULO "MODULO(A, B)" "CallStructure('modulo', (A, B))" "pysd.functions.modulo(A, B)" +IF THEN ELSE "IF THEN ELSE(A, B, C)" if_then_else "if_then_else(A, B, C)" "CallStructure('if_then_else', (A, B))" "pysd.functions.if_then_else(A, lambda: B, lambda: C)" + IF condition THEN value_true ELSE value_false IF A THEN B ELSE C "CallStructure('if_then_else', (A, B))" "pysd.functions.if_then_else(A, lambda: B, lambda: C)" +XIDZ "XIDZ(A, B, X)" safediv "safediv(A, B, X)" "CallStructure('xidz', (A, B, X))" "pysd.functions.xidz(A, B, X)" +ZIDZ "ZIDZ(A, B)" safediv "safediv(A, B)" "CallStructure('zidz', (A, B))" "pysd.functions.zidz(A, B)" + +VMIN VMIN(A) "CallStructure('vmin', (A,))" pysd.functions.vmin(A) +VMAX VMAX(A) "CallStructure('vmax', (A,))" pysd.functions.vmax(A) +SUM SUM(A) "CallStructure('sum', (A,))" pysd.functions.sum(A) +PROD PROD(A) "CallStructure('prod', (A,))" pysd.functions.prod(A) + +PULSE PULSE(start, width) "CallStructure('pulse', (start, width))" pysd.functions.pulse(start, width=width) + pulse pulse(magnitude, start) "CallStructure('Xpulse', (start, magnitude))" pysd.functions.pulse(start, magnitude=magnitude) Not tested for Xmile! + pulse pulse(magnitude, start, interval) "CallStructure('Xpulse_train', (start, interval, magnitude))" pysd.functions.pulse(start, repeat_time=interval, magnitude=magnitude) Not tested for Xmile! +PULSE TRAIN PULSE TRAIN(start, width, tbetween, end) "CallStructure('pulse_train', (start, tbetween, width, end))" pysd.functions.pulse(start, repeat_time=tbetween, width=width, end=end) +RAMP RAMP(slope, start_time, end_time) ramp ramp(slope, start_time, end_time) "CallStructure('ramp', (slope, start_time, end_time))" pysd.functions.ramp(time, slope, start_time, end_time) Not tested for Xmile! + ramp ramp(slope, start_time) "CallStructure('ramp', (slope, start_time))" pysd.functions.ramp(time, slope, start_time) Not tested for Xmile! +STEP STEP(height, step_time) step step(height, step_time) "CallStructure('step', (height, step_time))" pysd.functions.step(time, height, step_time) Not tested for Xmile! +GAME GAME(A) GameStructure(A) A +INITIAL INITIAL(value) init init(value) InitialStructure(value) pysd.statefuls.Initial +SAMPLE IF TRUE "SAMPLE IF TRUE(condition, input, initial_value)" "SampleIfTrueStructure(condition, input, initial_value)" pysd.statefuls.SampleIfTrue(...) diff --git a/docs/tables/get_functions.tab b/docs/tables/get_functions.tab new file mode 100644 index 00000000..996f490b --- /dev/null +++ b/docs/tables/get_functions.tab @@ -0,0 +1,9 @@ +Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments +GET XLS DATA "GET XLS DATA('file', 'sheet', 'time_row_or_col', 'cell')" "GetDataStructure('file', 'sheet', 'time_row_or_col', 'cell')" pysd.external.ExtData(...) +GET DIRECT DATA "GET DIRECT DATA('file', 'sheet', 'time_row_or_col', 'cell')" "GetDataStructure('file', 'sheet', 'time_row_or_col', 'cell')" pysd.external.ExtData(...) +GET XLS LOOKUPS "GET XLS LOOKUPS('file', 'sheet', 'x_row_or_col', 'cell')" "GetLookupsStructure('file', 'sheet', 'x_row_or_col', 'cell')" pysd.external.ExtLookup(...) +GET DIRECT LOOKUPS "GET DIRECT LOOKUPS('file', 'sheet', 'x_row_or_col', 'cell')" "GetLookupsStructure('file', 'sheet', 'x_row_or_col', 'cell')" pysd.external.ExtLookup(...) +GET XLS CONSTANTS "GET XLS CONSTANTS('file', 'sheet', 'cell')" "GetConstantsStructure('file', 'sheet', 'cell')" pysd.external.ExtConstant(...) +GET DIRECT CONSTANTS "GET DIRECT CONSTANTS('file', 'sheet', 'cell')" "GetConstantsStructure('file', 'sheet', 'cell')" pysd.external.ExtConstant(...) +GET XLS SUBSCRIPT "GET XLS SUBSCRIPT('file', 'sheet', 'first_cell', 'last_cell', 'prefix')" pysd.external.ExtSubscript(...) +GET DIRECT SUBSCRIPT "GET DIRECT SUBSCRIPT('file', 'sheet', 'first_cell', 'last_cell', 'prefix')" pysd.external.ExtSubscript(...) diff --git a/docs/tables/unary.tab b/docs/tables/unary.tab new file mode 100644 index 00000000..9d2cb6bc --- /dev/null +++ b/docs/tables/unary.tab @@ -0,0 +1,4 @@ +Vensim Vensim example Xmile Xmile example Abstract Syntax Python Translation Vensim comments Xmile comments Python comments +"\-" -A "\-" -A "LogicStructure(['negative'], (A,))" -A +"\+" +A "\+" +A A A +"\:NOT:" "\:NOT: A" not not A "LogicStructure([':NOT:'], (A,))" numpy.not(A) diff --git a/docs/tools.rst b/docs/tools.rst index 2a30bdfc..99aa61f5 100644 --- a/docs/tools.rst +++ b/docs/tools.rst @@ -1,7 +1,7 @@ Tools ===== -Some tools are given with the library. +Some additional tools are provided with the library. Benchmarking ------------ diff --git a/docs/whats_new.rst b/docs/whats_new.rst new file mode 100644 index 00000000..705c975a --- /dev/null +++ b/docs/whats_new.rst @@ -0,0 +1,66 @@ + +What's New +========== + +v3.0.0 (unreleased) +----------------------- + +New Features +~~~~~~~~~~~~ + +- The new :doc:`Abstract Model Representation ` translation and building workflow will allow to add new output languages in the future. +- Added new properties to the :py:class:`pysd.py_backend.model.Macro` to make more accessible some information: :py:attr:`.namespace`, :py:attr:`.subscripts`, :py:attr:`.dependencies`, :py:attr:`.modules`, :py:attr:`.doc`. +- Cleaner Python models: + - :py:data:`_namespace` and :py:data:`_dependencies` dictionaries have been removed from the file. + - Variables original names, dependencies metadata now are given through :py:meth:`pysd.py_backend.components.Component.add` decorator, instead of having them in the docstring. + - Merging of variable equations is now done using the coordinates to a pre-allocated array, instead of using the `magic` function :py:data:`pysd.py_backend.utils.xrmerge()`. + - Arranging and subseting arrays are now done inplace instead of using the magic function :py:data:`pysd.py_backend.utils.rearrange()`. + +Breaking changes +~~~~~~~~~~~~~~~~ + +- Set the argument :py:data:`flatten_output` from :py:meth:`.run` to :py:data:`True` by default. Previously it was set to :py:data:`False` by default. +- Move the docstring of the model to a property, :py:attr:`.doc`. Thus, it is not callable anymore. +- Allow the function :py:func:`pysd.py_backend.functions.pulse` to also perform the operations performed by :py:data:`pysd.py_backend.functions.pulse_train()` and :py:data:`pysd.py_backend.functions.pulse_magnitude()`. +- Change first argument of :py:func:`pysd.py_backend.functions.active_initial`, now it is the `stage of the model` and not the `time`. +- Simplify the function :py:data:`pysd.py_backend.utils.rearrange()` orienting it to perform simple rearrange cases for user interaction. +- Move :py:data:`pysd.py_backend.statefuls.Model` and :py:data:`pysd.py_backend.statefuls.Macro` to :py:class:`pysd.py_backend.model.Model` and :py:class:`pysd.py_backend.model.Macro`, respectively. +- Manage all kinds of lookups with the :py:class:`pysd.py_backend.lookups.Lookups` class. +- Include a second optional argument to lookups functions to set the final coordinates when a subscripted variable is passed as an argument. + +Deprecations +~~~~~~~~~~~~ + +- Remove :py:data:`pysd.py_backend.utils.xrmerge()`, :py:data:`pysd.py_backend.functions.pulse_train()`, :py:data:`pysd.py_backend.functions.pulse_magnitude()`, :py:data:`pysd.py_backend.functions.lookup()`, :py:data:`pysd.py_backend.functions.lookup_discrete()`, :py:data:`pysd.py_backend.functions.lookup_extrapolation()`, :py:data:`pysd.py_backend.functions.logical_and()`, :py:data:`pysd.py_backend.functions.logical_or()`, :py:data:`pysd.py_backend.functions.bounded_normal()`, :py:data:`pysd.py_backend.functions.log()`. +- Remove old translation and building files (:py:data:`pysd.translation`). + + +Bug fixes +~~~~~~~~~ + +- Generate the documentation of the model when loading it to avoid lossing information when replacing a variable value (:issue:`310`, :pull:`312`). +- Make random functions return arrays of the same shape as the variable, to avoid repeating values over a dimension (:issue:`309`, :pull:`312`). +- Fix bug when Vensim's :MACRO: definition is not at the top of the model file (:issue:`306`, :pull:`312`). +- Make builder identify the subscripts using a main range and subrange to allow using subscripts as numeric values as Vensim does (:issue:`296`, :issue:`301`, :pull:`312`). +- Fix bug of missmatching of functions and lookups names (:issue:`116`, :pull:`312`). +- Parse Xmile models case insensitively and ignoring the new lines characters (:issue:`203`, :issue:`253`, :pull:`312`). +- Add support for Vensim's `\:EXCEPT\: keyword `_ (:issue:`168`, :issue:`253`, :pull:`312`). +- Add spport for Xmile's FORCST and SAFEDIV functions (:issue:`154`, :pull:`312`). +- Add subscripts support for Xmile (:issue:`289`, :pull:`312`). +- Fix numeric error bug when using :py:data:`return_timestamps` and time step with non-integer values. + +Documentation +~~~~~~~~~~~~~ + +- Review the whole documentation, refract it, and describe the new features. + +Performance +~~~~~~~~~~~ + +- The variables defined in several equations are now assigned to a pre-allocated array instead of using :py:data:`pysd.py_backend.utils.xrmerge()`. +- The arranging and subseting of arrays is now done inplace instead of using the magic function :py:data:`pysd.py_backend.utils.rearrange()`. +- The grammars for Parsimonious are only compiled once per translation. + +Internal Changes +~~~~~~~~~~~~~~~~ +- The translation and the building of models has been totally modified to use the :doc:`Abstract Model Representation `. diff --git a/pysd/__init__.py b/pysd/__init__.py index 05ca59dd..e0fbe6d2 100644 --- a/pysd/__init__.py +++ b/pysd/__init__.py @@ -1,4 +1,4 @@ from .pysd import read_vensim, read_xmile, load from .py_backend import functions, statefuls, utils, external -from .py_backend.decorators import subs +from .py_backend.components import Component from ._version import __version__ diff --git a/pysd/_version.py b/pysd/_version.py index 62fa04d7..528787cf 100644 --- a/pysd/_version.py +++ b/pysd/_version.py @@ -1 +1 @@ -__version__ = "2.2.4" +__version__ = "3.0.0" diff --git a/pysd/translation/__init__.py b/pysd/builders/__init__.py similarity index 100% rename from pysd/translation/__init__.py rename to pysd/builders/__init__.py diff --git a/pysd/translation/vensim/__init__.py b/pysd/builders/python/__init__.py similarity index 100% rename from pysd/translation/vensim/__init__.py rename to pysd/builders/python/__init__.py diff --git a/pysd/builders/python/imports.py b/pysd/builders/python/imports.py new file mode 100644 index 00000000..92706918 --- /dev/null +++ b/pysd/builders/python/imports.py @@ -0,0 +1,80 @@ +from typing import Union + + +class ImportsManager(): + """ + Class to save the imported modules information for intelligent import + """ + _external_libs = {"numpy": "np", "xarray": "xr"} + _external_submodules = ["scipy"] + _internal_libs = [ + "functions", "statefuls", "external", "data", "lookups", "utils", + "model" + ] + + def __init__(self): + self._numpy, self._xarray = False, False + self._functions, self._statefuls, self._external, self._data,\ + self._lookups, self._utils, self._scipy, self._model =\ + set(), set(), set(), set(), set(), set(), set(), set() + + def add(self, module: str, function: Union[str, None] = None) -> None: + """ + Add a function from module. + + Parameters + ---------- + module: str + module name. + + function: str or None + function name. If None module will be set to true. + + """ + if function: + getattr(self, f"_{module}").add(function) + else: + setattr(self, f"_{module}", True) + + def get_header(self, outfile: str) -> str: + """ + Returns the importing information to print in the model file + + Parameters + ---------- + outfile: str + Name of the outfile to print in the header. + + Returns + ------- + text: str + Header of the translated model file. + + """ + text =\ + f'"""\nPython model \'{outfile}\'\nTranslated using PySD\n"""\n\n' + + text += "from pathlib import Path\n" + + for module, shortname in self._external_libs.items(): + if getattr(self, f"_{module}"): + text += f"import {module} as {shortname}\n" + + for module in self._external_submodules: + if getattr(self, f"_{module}"): + text += "from %(module)s import %(submodules)s\n" % { + "module": module, + "submodules": ", ".join(getattr(self, f"_{module}"))} + + text += "\n" + + for module in self._internal_libs: + if getattr(self, f"_{module}"): + text += "from pysd.py_backend.%(module)s import %(methods)s\n"\ + % { + "module": module, + "methods": ", ".join(getattr(self, f"_{module}"))} + + text += "from pysd import Component\n" + + return text diff --git a/pysd/builders/python/namespace.py b/pysd/builders/python/namespace.py new file mode 100644 index 00000000..5bd98eb4 --- /dev/null +++ b/pysd/builders/python/namespace.py @@ -0,0 +1,174 @@ +import re + +from unicodedata import normalize +from typing import List + +# used to create Python safe names with the variable reserved_words +from keyword import kwlist +from builtins import __dir__ as bidir +from pysd.py_backend.components import __dir__ as cdir +from pysd.py_backend.data import __dir__ as ddir +from pysd.py_backend.cache import __dir__ as cadir +from pysd.py_backend.external import __dir__ as edir +from pysd.py_backend.functions import __dir__ as fdir +from pysd.py_backend.statefuls import __dir__ as sdir +from pysd.py_backend.utils import __dir__ as udir + + +class NamespaceManager: + """ + NamespaceManager object allows includying new elements to the namespace + and searching for elements in the namespace. When includying new + elements a Python safe name is used to be able to write the equations. + + Parameters + ---------- + parameters: list (optional) + List of the parameters that are used as argument in the Macro. + By defaukt it is an empty list. + + """ + _reserved_words = set( + dir() + bidir() + cdir() + ddir() + cadir() + edir() + fdir() + + sdir() + udir()).union(kwlist) + + def __init__(self, parameters: List[str] = []): + self._used_words = self._reserved_words.copy() + # inlcude time to the namespace + self.namespace = {"Time": "time"} + # include time to the cleanspace (case and whitespace/underscore + # insensitive namespace) + self.cleanspace = {"time": "time"} + for parameter in parameters: + self.add_to_namespace(parameter) + + def add_to_namespace(self, string: str) -> None: + """ + Add a new string to the namespace. + + Parameters + ---------- + string: str + String to add to the namespace. + + Returns + ------- + None + + """ + self.make_python_identifier(string, add_to_namespace=True) + + def make_python_identifier(self, string: str, prefix: str = None, + add_to_namespace: bool = False) -> str: + """ + Takes an arbitrary string and creates a valid Python identifier. + + If the Python identifier created is already in the namespace, + but the input string is not (ie, two similar strings resolve to + the same Python identifier) or if the identifier is a reserved + word in the reserved_words list, or is a Python default + reserved word, adds _1, or if _1 is in the namespace, _2, etc. + + Parameters + ---------- + string: str + The text to be converted into a valid Python identifier. + + prefix: str or None (optional) + If given it will be used as a prefix for the output string. + Default is None. + + add_to_namespace: bool (optional) + If True it will add the passed string to the namespace and + to the cleanspace. Default is False. + + Returns + ------- + identifier: str + A vaild Python identifier based on the input string. + + Examples + -------- + >>> make_python_identifier('Capital') + 'capital' + + >>> make_python_identifier('multiple words') + 'multiple_words' + + >>> make_python_identifier('multiple spaces') + 'multiple_spaces' + + When the name is a Python keyword, add '_1' to differentiate it + >>> make_python_identifier('for') + 'for_1' + + Remove leading and trailing whitespace + >>> make_python_identifier(' whitespace ') + 'whitespace' + + Remove most special characters outright: + >>> make_python_identifier('H@t tr!ck') + 'ht_trck' + + add valid string to leading digits + >>> make_python_identifier('123abc') + 'nvs_123abc' + + already in namespace + >>> make_python_identifier('Var$') # namespace={'Var$': 'var'} + 'var' + + namespace conflicts + >>> make_python_identifier('Var@') # namespace={'Var$': 'var'} + 'var_1' + + >>> make_python_identifier('Var$') # namespace={'Var@': 'var', + ... 'Var%':'var_1'} + 'var_2' + + References + ---------- + Identifiers must follow the convention outlined here: + https://docs.python.org/2/reference/lexical_analysis.html#identifiers + + """ + s = string.lower() + clean_s = s.replace(" ", "_") + + # Make spaces into underscores + s = re.sub(r"[\s\t\n_]+", "_", s) + + # remove accents, diaeresis and others ó -> o + s = normalize("NFD", s).encode("ascii", "ignore").decode("utf-8") + + # Remove invalid characters + s = re.sub(r"[^0-9a-zA-Z_]", "", s) + + # If leading character is not a letter add nvs_. + # Only letters can be leading characters. + if prefix is not None: + s = prefix + "_" + s + elif re.findall(r"^[0-9]", s) or not s: + s = "nvs_" + s + elif re.findall(r"^_", s): + s = "nvs" + s + + # replace multiple _ after cleaning + s = re.sub(r"[_]+", "_", s) + + # Check that the string is not a Python identifier + identifier = s + i = 1 + while identifier in self._used_words: + identifier = s + '_' + str(i) + i += 1 + + # include the word in used words to avoid using it againg + self._used_words.add(identifier) + + if add_to_namespace: + # include word to the namespace + self.namespace[string] = identifier + self.cleanspace[clean_s] = identifier + + return identifier diff --git a/pysd/builders/python/python_expressions_builder.py b/pysd/builders/python/python_expressions_builder.py new file mode 100644 index 00000000..dca8e2e4 --- /dev/null +++ b/pysd/builders/python/python_expressions_builder.py @@ -0,0 +1,2102 @@ +""" +The translation from Abstract Syntax Tree to Python happens in both ways. +The outer expression is visited with its builder, which will split its +arguments and visit them with their respective builders. Once the lowest +level is reached, it will be translated into Python returning a BuildAST +object, this object will include the python expression, its subscripts, +its calls to other and its arithmetic order (see Build AST for more info). +BuildAST will be returned for each visited argument from the lower +lever to the top level, giving the final expression. +""" +import warnings +from dataclasses import dataclass +from typing import Union + +import numpy as np +from pysd.py_backend.utils import compute_shape + +from pysd.translators.structures.abstract_expressions import\ + AbstractSyntax, ArithmeticStructure, CallStructure, DataStructure,\ + DelayFixedStructure, DelayStructure, DelayNStructure, ForecastStructure,\ + GameStructure, GetConstantsStructure, GetDataStructure,\ + GetLookupsStructure, InitialStructure, InlineLookupsStructure,\ + IntegStructure, LogicStructure, LookupsStructure, ReferenceStructure,\ + SampleIfTrueStructure, SmoothNStructure, SmoothStructure,\ + SubscriptsReferenceStructure, TrendStructure + +from .python_functions import functionspace +from .subscripts import SubscriptManager + + +@dataclass +class BuildAST: + """ + Python expression holder. + + Parameters + ---------- + expression: str + The Python expression. + calls: dict + The calls to other variables for the dependencies dictionary. + subscripts: dict + The subscripts dict of the expression. + order: int + Arithmetic order of the expression. The arithmetic order depends + on the last arithmetic operation. If the expression is a number, + a call to a function, or is between parenthesis; its order will + be 0. If the expression its an exponential of two terms its order + will be 1. If the expression is a product or division its order + will be 2. If the expression is a sum or substraction its order + will be 3. If the expression is a logical comparison its order + will be 4. + + """ + expression: str + calls: dict + subscripts: dict + order: int + + def __str__(self) -> str: + # makes easier building + return self.expression + + def reshape(self, subscripts: SubscriptManager, + final_subscripts: dict, + final_element: bool = False) -> None: + """ + Reshape the object to the desired subscripts. It will modify the + expression and lower the order if it is not 0. + + Parameters + ---------- + subscripts: SubscriptManager + The subscripts of the section. + final_subscripts: dict + The desired final subscripts. + final_element: bool (optional) + If True the array will be reshaped with the final subscripts + to have the shame shape. Otherwise, a length 1 dimension + will be included in the position to allow arithmetic + operations with other arrays. Default is False. + + """ + if not final_subscripts or ( + self.subscripts == final_subscripts + and list(self.subscripts) == list(final_subscripts)): + # Same dictionary in the same order, do nothing + pass + elif not self.subscripts: + # Original expression is not an array + # NUMPY: object.expression = np.full(%s, %(shape)s) + subscripts_out = subscripts.simplify_subscript_input( + final_subscripts)[1] + self.expression = "xr.DataArray(%s, %s, %s)" % ( + self.expression, subscripts_out, list(final_subscripts) + ) + self.order = 0 + self.subscripts = final_subscripts + else: + # Original expression is an array + self.lower_order(-1) + + # Reorder subscrips + final_order = { + sub: self.subscripts[sub] + for sub in final_subscripts + if sub in self.subscripts + } + if list(final_order) != list(self.subscripts): + # NUMPY: reorder dims if neccessary with np.moveaxis or similar + self.expression +=\ + f".transpose({', '.join(map(repr, final_order))})" + self.subscripts = final_order + + # add new dimensions + if final_element and final_subscripts != self.subscripts: + # NUMPY: remove final_element condition from top + # NUMPY: add new axis with [:, None, :] + # NUMPY: move final_element condition here and use np.tile + for i, dim in enumerate(final_subscripts): + if dim not in self.subscripts: + subscripts_out = subscripts.simplify_subscript_input( + {dim: final_subscripts[dim]})[1] + self.expression +=\ + f".expand_dims({subscripts_out}, {i})" + + self.subscripts = final_subscripts + + def lower_order(self, new_order: int) -> None: + """ + Lower the order to maintain the correct order in arithmetic + operations. If the requestes order is smaller than the current + order parenthesis will be added to the expression to lower its + order to 0. + + Parameters + ---------- + new_order: int + The required new order of the expression. If 0 it will be + assumed that the expression will be passed as an argument + of a function and therefore no operations will be done. If + order 0 is required, a negative value can be used for + new_order. + + """ + if self.order >= new_order and self.order != 0 and new_order != 0: + # if current operator order is 0 do not need to do anything + # if the order of operations conflicts add parenthesis + # if new order is 0 do not need to do anything, as it may be + # an argument to a function. To force the 0 order a negative + # value can be used, which will force the parenthesis + # (necessary to reshape some arrays) + self.expression = "(%s)" % self.expression + self.order = 0 + + +class StructureBuilder: + """ + Main builder for Abstract Syntax Tree structures. All the builders + are children of this class, which allows them inheriting the methods. + """ + def __init__(self, value: object, component: object): + # component typing should be ComponentBuilder, but importing it + # for typing would create a circular dependency :S + self.value = value + self.arguments = {} + self.component = component + self.element = component.element + self.section = component.section + self.def_subs = component.subscripts_dict + + @staticmethod + def join_calls(arguments: dict) -> dict: + """ + Merge the calls of the arguments. + + Parameters + ---------- + arguments: dict + The dictionary of arguments. The keys should br strings of + ordered integer numbers starting from 0. + + Returns + ------- + calls: dict + The merged dictionary of calls. + + """ + if len(arguments) == 0: + # No arguments + return {} + elif len(arguments) == 1: + # Only one argument + return arguments["0"].calls + else: + # Several arguments + return merge_dependencies( + *[val.calls for val in arguments.values()]) + + def reorder(self, arguments: dict, force: bool = None) -> dict: + """ + Reorder the subscripts of the arguments to make them match. + + Parameters + ---------- + arguments: dict + The dictionary of arguments. The keys should br strings of + ordered integer numbers starting from 0. + force: 'component', 'equal', or None (optional) + If force is 'component' it will force the arguments to have + the subscripts of the component definition. If force is + 'equal' it will force all the arguments to have the same + subscripts, includying the floats. If force is None, it + will only modify the shape of the arrays adding length 1 + dimensions to allow operation between different shape arrays. + Default is None. + + Returns + ------- + final_subscripts: dict + The final_subscripts after reordering all the elements. + + """ + if force == "component": + final_subscripts = self.def_subs or {} + else: + final_subscripts = self.get_final_subscripts(arguments) + + [arguments[key].reshape( + self.section.subscripts, final_subscripts, bool(force)) + for key in arguments + if arguments[key].subscripts or force == "equal"] + + return final_subscripts + + def get_final_subscripts(self, arguments: dict) -> dict: + """ + Get the final subscripts of a combination of arguments. + + Parameters + ---------- + arguments: dict + The dictionary of arguments. The keys should br strings of + ordered integer numbers starting from 0. + + Returns + ------- + final_subscripts: dict + The final_subscripts of combining all the elements. + + """ + if len(arguments) == 0: + return {} + elif len(arguments) == 1: + return arguments["0"].subscripts + else: + return self._compute_final_subscripts( + [arg.subscripts for arg in arguments.values()]) + + def _compute_final_subscripts(self, subscripts_list: list) -> dict: + """ + Compute final subscripts from a list of subscript dictionaries. + + Parameters + ---------- + subscript_list: list of dicts + List of subscript dictionaries. + + """ + expression = {} + [expression.update(subscript) + for subscript in subscripts_list if subscript] + # TODO reorder final_subscripts taking into account def_subs + # this way try to minimize the reordering operations + return expression + + def update_object_subscripts(self, name: str, + component_final_subs: dict) -> None: + """ + Update the object subscripts. Needed for those objects that + use 'add' method to load several components at once. + + Parameters + ---------- + name: str + The name of the object in the objects dictionary from the + element. + component_final_subs: dict + The subscripts of the component but with the element + subscript ranges as keys. This can differ from the component + subscripts when the component is defined with subranges of + the final subscript ranges. + + """ + # Get the component used to define the object first time + origin_comp = self.element.objects[name]["component"] + if isinstance(origin_comp.subscripts_dict, dict): + # The original component subscript dictionary is a dict + if len(list(origin_comp.subscripts_dict)) == 1: + # If the subscript dict has only one dimension + # all the components can be loaded in 1D array directly + # with the same length as the given by the sum of the + # components + key = list(origin_comp.subscripts_dict.keys())[0] + value = list(component_final_subs.values())[0] + origin_comp.subscripts_dict[key] += value + self.element.objects[name]["final_subs"] =\ + origin_comp.subscripts_dict + else: + # If the subscripts dict has several dimensions, then + # a multi-dimensional array needs to be computed, + # in some cases, when mixed definitions are used in an + # element (e.g. GET DIRECT CONSTANTS and regular constants), + # this array can have some empty subarrays, therefore a + # list should be created and manage the loaded data + # with manage_multi_def in the element building + origin_comp.subscripts_dict = [origin_comp.subscripts_dict] + self.element.objects[name]["final_subs"] =\ + self.element.subs_dict + if isinstance(origin_comp.subscripts_dict, list): + # The original component subscript dictionary is a list + # (this happens when other components have already been + # added with 'add' method) + origin_comp.subscripts_dict.append(component_final_subs) + + +class OperationBuilder(StructureBuilder): + """Builder for arithmetic and logical operations.""" + _operators_build = { + "^": ("%(left)s**%(right)s", None, 1), + "*": ("%(left)s*%(right)s", None, 2), + "/": ("%(left)s/%(right)s", None, 2), + "+": ("%(left)s + %(right)s", None, 3), + "-": ("%(left)s - %(right)s", None, 3), + "=": ("%(left)s == %(right)s", None, 4), + "<>": ("%(left)s != %(right)s", None, 4), + ">=": ("%(left)s >= %(right)s", None, 4), + ">": ("%(left)s > %(right)s", None, 4), + "<=": ("%(left)s <= %(right)s", None, 4), + "<": ("%(left)s < %(right)s", None, 4), + ":NOT:": ("np.logical_not(%s)", ("numpy",), 0), + ":AND:": ("np.logical_and(%(left)s, %(right)s)", ("numpy",), 0), + ":OR:": ("np.logical_or(%(left)s, %(right)s)", ("numpy",), 0), + "negative": ("-%s", None, 3), + } + + def __init__(self, operation: Union[ArithmeticStructure, LogicStructure], + component: object): + super().__init__(None, component) + self.operators = operation.operators.copy() + self.arguments = { + str(i): arg for i, arg in enumerate(operation.arguments)} + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + operands = {} + calls = self.join_calls(arguments) + final_subscripts = self.reorder(arguments) + arguments = [arguments[str(i)] for i in range(len(arguments))] + dependencies, order = self._operators_build[self.operators[-1]][1:] + + if dependencies: + # Add necessary dependencies to the imports + self.section.imports.add(*dependencies) + + if self.operators[-1] == "^": + # Right side of the exponential can be from higher order + arguments[-1].lower_order(2) + else: + arguments[-1].lower_order(order) + + if len(arguments) == 1: + # not and negative operations (only 1 element) + if self.operators[0] == "negative": + order = 1 + expression = self._operators_build[self.operators[0]][0] + return BuildAST( + expression=expression % arguments[0], + calls=calls, + subscripts=final_subscripts, + order=order) + + # Add the arguments to the expression with the operator, + # they are built from right to left + # Get the last argument as the RHS of the first operation + operands["right"] = arguments.pop() + while arguments or self.operators: + # Get the operator and the LHS of the operation + expression = self._operators_build[self.operators.pop()][0] + operands["left"] = arguments.pop() + # Lower the order of the LHS if neccessary + operands["left"].lower_order(order) + # Include the operation in the RHS for next iteration + operands["right"] = expression % operands + + return BuildAST( + expression=operands["right"], + calls=calls, + subscripts=final_subscripts, + order=order) + + +class GameBuilder(StructureBuilder): + """Builder for GAME expressions.""" + def __init__(self, game_str: GameStructure, component: object): + super().__init__(None, component) + self.arguments = {"expr": game_str.expression} + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + # Game calls are ignored as we have no support for a similar + # feature, we simpli return the content inside the GAME call + return arguments["expr"] + + +class CallBuilder(StructureBuilder): + """Builder for calls to functions, macros and lookups.""" + def __init__(self, call_str: CallStructure, component: object): + super().__init__(None, component) + function_name = call_str.function.reference + self.arguments = { + str(i): arg for i, arg in enumerate(call_str.arguments)} + + if function_name in self.section.macrospace: + # Build macro + self.macro_name = function_name + self.build = self.build_macro_call + elif function_name in self.section.namespace.cleanspace: + # Build lookupcall + self.arguments["function"] = call_str.function + self.build = self.build_lookups_call + elif function_name in functionspace: + # Build direct function + self.function = function_name + self.build = self.build_function_call + elif function_name == "a_function_of": + # Build incomplete function + self.build = self.build_incomplete_call + else: + # Build missing function + self.function = function_name + self.build = self.build_not_implemented + + def build_not_implemented(self, arguments: dict) -> BuildAST: + """ + Build method for not implemented function calls. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + final_subscripts = self.reorder(arguments) + warnings.warn( + "\n\nTrying to translate '" + + self.function.upper().replace("_", " ") + + "' which it is not implemented on PySD. The translated " + + "model will crash... " + ) + self.section.imports.add("functions", "not_implemented_function") + + return BuildAST( + expression="not_implemented_function('%s', %s)" % ( + self.function, + ", ".join(arg.expression for arg in arguments.values())), + calls=self.join_calls(arguments), + subscripts=final_subscripts, + order=0) + + def build_incomplete_call(self, arguments: dict) -> BuildAST: + """ + Build method for incomplete function calls. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + warnings.warn( + "'%s' has no equation specified" % self.element.name, + SyntaxWarning, stacklevel=2 + ) + self.section.imports.add("functions", "incomplete") + return BuildAST( + expression="incomplete(%s)" % ", ".join( + arg.expression for arg in arguments.values()), + calls=self.join_calls(arguments), + subscripts=self.def_subs, + order=0) + + def build_macro_call(self, arguments: dict) -> BuildAST: + """ + Build method for macro calls. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.section.imports.add("model", "Macro") + # Get macro from macrospace + macro = self.section.macrospace[self.macro_name] + + calls = self.join_calls(arguments) + final_subscripts = self.reorder(arguments) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.macro_name + "_" + self.element.identifier, prefix="_macro") + arguments["file"] = macro.path.name + arguments["macro_name"] = macro.name + arguments["args"] = "{%s}" % ", ".join([ + "'%s': lambda: %s" % (key, val) + for key, val in zip(macro.params, arguments.values()) + ]) + + # Create Macro object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Macro(_root.joinpath('%(file)s'), " + "%(args)s, '%(macro_name)s', " + "time_initialization=lambda: __data['time'], " + "py_name='%(name)s')" % arguments, + } + # Add other_dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": calls, + "step": calls + } + + return BuildAST( + expression="%s()" % arguments["name"], + calls={arguments["name"]: 1}, + subscripts=final_subscripts, + order=0) + + def build_lookups_call(self, arguments: dict) -> BuildAST: + """ + Build method for loookups calls. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + if arguments["0"].subscripts: + # Build lookups with subcripted arguments + # it is neccessary to give the final subscripts information + # in the call to rearrange it correctly + final_subscripts =\ + self.get_final_subscripts(arguments) + expression = arguments["function"].expression.replace( + "()", f"(%(0)s, {final_subscripts})") + else: + # Build lookups with float arguments + final_subscripts = arguments["function"].subscripts + expression = arguments["function"].expression.replace( + "()", "(%(0)s)") + + # NUMPY: we need to manage inside lookup with subscript and later + # return the values in a correct ndarray + return BuildAST( + expression=expression % arguments, + calls=self.join_calls(arguments), + subscripts=final_subscripts, + order=0) + + def build_function_call(self, arguments: dict) -> BuildAST: + """ + Build method for function calls. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + # Get the function expression from the functionspace + expression, modules = functionspace[self.function] + if modules: + # Update module dependencies in imports + self.section.imports.add(*modules) + + calls = self.join_calls(arguments) + + if "__data['time']" in expression: + # If the expression depens on time add to the dependencies + merge_dependencies(calls, {"time": 1}, inplace=True) + + if "%(axis)s" in expression: + # Vectorial expressions, compute the axis using dimensions + # with ! operator + final_subscripts, arguments["axis"] = self._compute_axis(arguments) + + elif "%(size)s" in expression: + # Random expressions, need to give the final size of the + # component to create one value per final coordinate + final_subscripts = self.reorder(arguments, force="component") + arguments["size"] = tuple(compute_shape(final_subscripts)) + if arguments["size"]: + # Create an xarray from the random function output + # NUMPY: not necessary + # generate an xarray from the output + subs = self.section.subscripts.simplify_subscript_input( + self.def_subs)[1] + expression = f"xr.DataArray({expression}, {subs}, "\ + f"{list(self.def_subs)})" + + elif self.function == "active_initial": + # Ee need to ensure that active initial outputs are always the + # same and update dependencies as stateful object + name = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_active_initial") + final_subscripts = self.reorder(arguments, force="equal") + self.element.other_dependencies[name] = { + "initial": arguments["1"].calls, + "step": arguments["0"].calls + } + + calls = {name: 1} + else: + final_subscripts = self.reorder(arguments) + if self.function == "xidz" and final_subscripts: + # xidz must always return the same shape object + if not arguments["1"].subscripts: + [arguments[i].reshape( + self.section.subscripts, final_subscripts, True) + for i in ["0", "1"]] + elif arguments["0"].subscripts or arguments["2"].subscripts: + # NUMPY: not need this statement + [arguments[i].reshape( + self.section.subscripts, final_subscripts, True) + for i in ["0", "1", "2"] + if arguments[i].subscripts] + elif self.function == "zidz" and final_subscripts: + # zidz must always return the same shape object + arguments["0"].reshape( + self.section.subscripts, final_subscripts, True) + if arguments["1"].subscripts: + # NUMPY: not need this statement + arguments["1"].reshape( + self.section.subscripts, final_subscripts, True) + elif self.function == "if_then_else" and final_subscripts: + # if_then_else must always return the same shape object + if not arguments["0"].subscripts: + # condition is a float + [arguments[i].reshape( + self.section.subscripts, final_subscripts, True) + for i in ["1", "2"]] + else: + # condition has dimensions + [arguments[i].reshape( + self.section.subscripts, final_subscripts, True) + for i in ["0", "1", "2"]] + + return BuildAST( + expression=expression % arguments, + calls=calls, + subscripts=final_subscripts, + order=0) + + def _compute_axis(self, arguments: dict) -> tuple: + """ + Compute the axis to apply a vectorial function. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + coords: dict + The final coordinates after executing the vectorial function + axis: list + The list of dimensions to apply the function. Uses the + dimensions with "!" at the end. + + """ + subscripts = arguments["0"].subscripts + axis = [] + coords = {} + for subs in subscripts: + if subs.endswith("!"): + # dimensions to apply along + axis.append(subs) + else: + # dimensions remaining + coords[subs] = subscripts[subs] + return coords, axis + + +class ExtLookupBuilder(StructureBuilder): + """Builder for External Lookups.""" + def __init__(self, getlookup_str: GetLookupsStructure, component: object): + super().__init__(None, component) + self.file = getlookup_str.file + self.tab = getlookup_str.tab + self.x_row_or_col = getlookup_str.x_row_or_col + self.cell = getlookup_str.cell + self.arguments = {} + + def build(self, arguments: dict) -> Union[BuildAST, None]: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST or None + The built object, unless the component has been added to an + existing object using the 'add' method. + + """ + self.component.type = "Lookup" + self.component.subtype = "External" + arguments["params"] = "'%s', '%s', '%s', '%s'" % ( + self.file, self.tab, self.x_row_or_col, self.cell + ) + final_subs, arguments["subscripts"] =\ + self.section.subscripts.simplify_subscript_input( + self.def_subs, self.element.subscripts) + + if "ext_lookups" in self.element.objects: + # Object already exists, use 'add' method + self.element.objects["ext_lookups"]["expression"] += "\n\n"\ + + self.element.objects["ext_lookups"]["name"]\ + + ".add(%(params)s, %(subscripts)s)" % arguments + + self.update_object_subscripts("ext_lookups", final_subs) + + return None + else: + # Create a new object + self.section.imports.add("external", "ExtLookup") + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_ext_lookup") + arguments["final_subs"] = "%(final_subs)s" + self.component.subscripts_dict = final_subs + + self.element.objects["ext_lookups"] = { + "name": arguments["name"], + "expression": "%(name)s = ExtLookup(%(params)s, " + "%(subscripts)s, _root, " + "%(final_subs)s , '%(name)s')" % arguments, + "component": self.component, + "final_subs": final_subs + } + + return BuildAST( + expression=arguments["name"] + "(x, final_subs)", + calls={ + "__external__": arguments["name"], + "__lookup__": arguments["name"] + }, + subscripts=final_subs, + order=0) + + +class ExtDataBuilder(StructureBuilder): + """Builder for External Data.""" + def __init__(self, getdata_str: GetDataStructure, component: object): + super().__init__(None, component) + self.file = getdata_str.file + self.tab = getdata_str.tab + self.time_row_or_col = getdata_str.time_row_or_col + self.cell = getdata_str.cell + self.keyword = component.keyword + self.arguments = {} + + def build(self, arguments: dict) -> Union[BuildAST, None]: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST or None + The built object, unless the component has been added to an + existing object using the 'add' method. + + """ + self.component.type = "Data" + self.component.subtype = "External" + arguments["params"] = "'%s', '%s', '%s', '%s'" % ( + self.file, self.tab, self.time_row_or_col, self.cell + ) + final_subs, arguments["subscripts"] =\ + self.section.subscripts.simplify_subscript_input( + self.def_subs, self.element.subscripts) + arguments["method"] = "'%s'" % self.keyword if self.keyword else None + + if "ext_data" in self.element.objects: + # Object already exists, use add method + self.element.objects["ext_data"]["expression"] += "\n\n"\ + + self.element.objects["ext_data"]["name"]\ + + ".add(%(params)s, %(method)s, %(subscripts)s)" % arguments + + self.update_object_subscripts("ext_data", final_subs) + + return None + else: + # Create a new object + self.section.imports.add("external", "ExtData") + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_ext_data") + arguments["final_subs"] = "%(final_subs)s" + self.component.subscripts_dict = final_subs + + self.element.objects["ext_data"] = { + "name": arguments["name"], + "expression": "%(name)s = ExtData(%(params)s, " + " %(method)s, %(subscripts)s, " + "_root, %(final_subs)s ,'%(name)s')" % arguments, + "component": self.component, + "final_subs": final_subs + } + + return BuildAST( + expression=arguments["name"] + "(time())", + calls={ + "__external__": arguments["name"], + "__data__": arguments["name"], + "time": 1}, + subscripts=final_subs, + order=0) + + +class ExtConstantBuilder(StructureBuilder): + """Builder for External Constants.""" + def __init__(self, getconstant_str: GetConstantsStructure, + component: object): + super().__init__(None, component) + self.file = getconstant_str.file + self.tab = getconstant_str.tab + self.cell = getconstant_str.cell + self.arguments = {} + + def build(self, arguments: dict) -> Union[BuildAST, None]: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST or None + The built object, unless the component has been added to an + existing object using the 'add' method. + + """ + self.component.type = "Constant" + self.component.subtype = "External" + arguments["params"] = "'%s', '%s', '%s'" % ( + self.file, self.tab, self.cell + ) + final_subs, arguments["subscripts"] =\ + self.section.subscripts.simplify_subscript_input( + self.def_subs, self.element.subscripts) + + if "constants" in self.element.objects: + # Object already exists, use 'add' method + self.element.objects["constants"]["expression"] += "\n\n"\ + + self.element.objects["constants"]["name"]\ + + ".add(%(params)s, %(subscripts)s)" % arguments + + self.update_object_subscripts("constants", final_subs) + + return None + else: + # Create a new object + self.section.imports.add("external", "ExtConstant") + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_ext_constant") + arguments["final_subs"] = "%(final_subs)s" + self.component.subscripts_dict = final_subs + + self.element.objects["constants"] = { + "name": arguments["name"], + "expression": "%(name)s = ExtConstant(%(params)s, " + "%(subscripts)s, _root, %(final_subs)s, " + "'%(name)s')" % arguments, + "component": self.component, + "final_subs": final_subs + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={"__external__": arguments["name"]}, + subscripts=final_subs, + order=0) + + +class TabDataBuilder(StructureBuilder): + """Builder for empty DATA expressions.""" + def __init__(self, data_str: DataStructure, component: object): + super().__init__(None, component) + self.keyword = component.keyword + self.arguments = {} + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.section.imports.add("data", "TabData") + + final_subs, arguments["subscripts"] =\ + self.section.subscripts.simplify_subscript_input( + self.def_subs, self.element.subscripts) + + arguments["real_name"] = self.element.name + arguments["py_name"] =\ + self.section.namespace.namespace[self.element.name] + arguments["subscripts"] = self.def_subs + arguments["method"] = "'%s'" % self.keyword if self.keyword else None + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_data") + + # Create TabData object + self.element.objects["tab_data"] = { + "name": arguments["name"], + "expression": "%(name)s = TabData('%(real_name)s', '%(py_name)s', " + "%(subscripts)s, %(method)s)" % arguments + } + + return BuildAST( + expression=arguments["name"] + "(time())", + calls={"time": 1, "__data__": arguments["name"]}, + subscripts=final_subs, + order=0) + + +class InitialBuilder(StructureBuilder): + """Builder for Initials.""" + def __init__(self, initial_str: InitialStructure, component: object): + super().__init__(None, component) + self.arguments = { + "initial": initial_str.initial + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Initial" + self.section.imports.add("statefuls", "Initial") + + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_initial") + + # Create the object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Initial(lambda: %(initial)s, " + "'%(name)s')" % arguments, + } + # Add other-dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": arguments["initial"].calls, + "step": {} + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class IntegBuilder(StructureBuilder): + """Builder for Integs/Stocks.""" + def __init__(self, integ_str: IntegStructure, component: object): + super().__init__(None, component) + self.arguments = { + "flow": integ_str.flow, + "initial": integ_str.initial + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Integ" + self.section.imports.add("statefuls", "Integ") + + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["flow"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_integ") + + # Create the object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Integ(lambda: %(flow)s, " + "lambda: %(initial)s, '%(name)s')" % arguments + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": arguments["initial"].calls, + "step": arguments["flow"].calls + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class DelayBuilder(StructureBuilder): + """Builder for regular Delays.""" + def __init__(self, dtype: str, + delay_str: Union[DelayStructure, DelayNStructure], + component: object): + super().__init__(None, component) + self.arguments = { + "input": delay_str.input, + "delay_time": delay_str.delay_time, + "initial": delay_str.initial, + "order": delay_str.order + } + self.dtype = dtype + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Delay" + self.section.imports.add("statefuls", self.dtype) + + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["delay_time"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix=f"_{self.dtype.lower()}") + arguments["dtype"] = self.dtype + + # Add the object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = %(dtype)s(lambda: %(input)s, " + "lambda: %(delay_time)s, lambda: %(initial)s, " + "lambda: %(order)s, " + "time_step, '%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": merge_dependencies( + arguments["initial"].calls, + arguments["delay_time"].calls, + arguments["order"].calls), + "step": merge_dependencies( + arguments["input"].calls, + arguments["delay_time"].calls) + + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class DelayFixedBuilder(StructureBuilder): + """Builder for Delay Fixed.""" + def __init__(self, delay_str: DelayFixedStructure, component: object): + super().__init__(None, component) + self.arguments = { + "input": delay_str.input, + "delay_time": delay_str.delay_time, + "initial": delay_str.initial, + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "DelayFixed" + self.section.imports.add("statefuls", "DelayFixed") + + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_delayfixed") + + # Create object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = DelayFixed(lambda: %(input)s, " + "lambda: %(delay_time)s, lambda: %(initial)s, " + "time_step, '%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": merge_dependencies( + arguments["initial"].calls, + arguments["delay_time"].calls), + "step": arguments["input"].calls + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class SmoothBuilder(StructureBuilder): + """Builder for Smooths.""" + def __init__(self, smooth_str: Union[SmoothStructure, SmoothNStructure], + component: object): + super().__init__(None, component) + self.arguments = { + "input": smooth_str.input, + "smooth_time": smooth_str.smooth_time, + "initial": smooth_str.initial, + "order": smooth_str.order + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Smooth" + self.section.imports.add("statefuls", "Smooth") + + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["smooth_time"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_smooth") + + # TODO in the future we need to ad timestep to show warnings about + # the smooth time as its done with delays (see vensim help for smooth) + # TODO in the future we may want to have 2 py_backend classes for + # smooth as the behaviour is different for SMOOTH and SMOOTH N when + # using RingeKutta scheme + + # Create object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Smooth(lambda: %(input)s, " + "lambda: %(smooth_time)s, lambda: %(initial)s, " + "lambda: %(order)s, '%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": merge_dependencies( + arguments["initial"].calls, + arguments["smooth_time"].calls, + arguments["order"].calls), + "step": merge_dependencies( + arguments["input"].calls, + arguments["smooth_time"].calls) + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class TrendBuilder(StructureBuilder): + """Builder for Trends.""" + def __init__(self, trend_str: TrendStructure, component: object): + super().__init__(None, component) + self.arguments = { + "input": trend_str.input, + "average_time": trend_str.average_time, + "initial_trend": trend_str.initial_trend, + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Trend" + self.section.imports.add("statefuls", "Trend") + + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["average_time"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial_trend"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_trend") + + # Create object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Trend(lambda: %(input)s, " + "lambda: %(average_time)s, " + "lambda: %(initial_trend)s, " + "'%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": merge_dependencies( + arguments["initial_trend"].calls, + arguments["input"].calls, + arguments["average_time"].calls), + "step": merge_dependencies( + arguments["input"].calls, + arguments["average_time"].calls) + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class ForecastBuilder(StructureBuilder): + """Builder for Forecasts.""" + def __init__(self, forecast_str: ForecastStructure, component: object): + super().__init__(None, component) + self.arguments = { + "input": forecast_str.input, + "average_time": forecast_str.average_time, + "horizon": forecast_str.horizon, + "initial_trend": forecast_str.initial_trend + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "Forecast" + self.section.imports.add("statefuls", "Forecast") + + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["average_time"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["horizon"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial_trend"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_forecast") + + # Create object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = Forecast(lambda: %(input)s, " + "lambda: %(average_time)s, lambda: %(horizon)s, " + "lambda: %(initial_trend)s, '%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": merge_dependencies( + arguments["input"].calls, + arguments["initial_trend"].calls), + "step": merge_dependencies( + arguments["input"].calls, + arguments["average_time"].calls, + arguments["horizon"].calls) + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class SampleIfTrueBuilder(StructureBuilder): + """Builder for Sample If True.""" + def __init__(self, sampleiftrue_str: SampleIfTrueStructure, + component: object): + super().__init__(None, component) + self.arguments = { + "condition": sampleiftrue_str.condition, + "input": sampleiftrue_str.input, + "initial": sampleiftrue_str.initial, + } + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Stateful" + self.component.subtype = "SampleIfTrue" + self.section.imports.add("statefuls", "SampleIfTrue") + + arguments["condition"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["input"].reshape( + self.section.subscripts, self.def_subs, True) + arguments["initial"].reshape( + self.section.subscripts, self.def_subs, True) + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_sampleiftrue") + + # Create object + self.element.objects[arguments["name"]] = { + "name": arguments["name"], + "expression": "%(name)s = SampleIfTrue(lambda: %(condition)s, " + "lambda: %(input)s, lambda: %(initial)s, " + "'%(name)s')" % arguments, + } + # Add other dependencies + self.element.other_dependencies[arguments["name"]] = { + "initial": + arguments["initial"].calls, + "step": merge_dependencies( + arguments["condition"].calls, + arguments["input"].calls) + } + + return BuildAST( + expression=arguments["name"] + "()", + calls={arguments["name"]: 1}, + subscripts=self.def_subs, + order=0) + + +class LookupsBuilder(StructureBuilder): + """Builder for regular Lookups.""" + def __init__(self, lookups_str: LookupsStructure, component: object): + super().__init__(None, component) + self.arguments = {} + self.x = lookups_str.x + self.y = lookups_str.y + self.keyword = lookups_str.type + + def build(self, arguments: dict) -> Union[BuildAST, None]: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST or None + The built object, unless the component has been added to an + existing object using the 'add' method. + + """ + self.component.type = "Lookup" + self.component.subtype = "Normal" + # Get the numeric values as numpy arrays + arguments["x"] = np.array2string( + np.array(self.x), + separator=",", + threshold=len(self.x) + ) + arguments["y"] = np.array2string( + np.array(self.y), + separator=",", + threshold=len(self.y) + ) + arguments["subscripts"] = self.def_subs + arguments["interp"] = self.keyword + + if "hardcoded_lookups" in self.element.objects: + # Object already exists, use 'add' method + self.element.objects["hardcoded_lookups"]["expression"] += "\n\n"\ + + self.element.objects["hardcoded_lookups"]["name"]\ + + ".add(%(x)s, %(y)s, %(subscripts)s)" % arguments + + return None + else: + # Create a new object + self.section.imports.add("lookups", "HardcodedLookups") + + arguments["name"] = self.section.namespace.make_python_identifier( + self.element.identifier, prefix="_hardcodedlookup") + + arguments["final_subs"] = self.element.subs_dict + + self.element.objects["hardcoded_lookups"] = { + "name": arguments["name"], + "expression": "%(name)s = HardcodedLookups(%(x)s, %(y)s, " + "%(subscripts)s, '%(interp)s', " + "%(final_subs)s, '%(name)s')" + % arguments + } + + return BuildAST( + expression=arguments["name"] + "(x, final_subs)", + calls={"__lookup__": arguments["name"]}, + subscripts=self.def_subs, + order=0) + + +class InlineLookupsBuilder(StructureBuilder): + """Builder for inline Lookups.""" + def __init__(self, inlinelookups_str: InlineLookupsStructure, + component: object): + super().__init__(None, component) + self.arguments = { + "value": inlinelookups_str.argument + } + self.lookups = inlinelookups_str.lookups + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.component.type = "Auxiliary" + self.component.subtype = "with Lookup" + self.section.imports.add("numpy") + # Get the numeric values as numpy arrays + arguments["x"] = np.array2string( + np.array(self.lookups.x), + separator=",", + threshold=len(self.lookups.x) + ) + arguments["y"] = np.array2string( + np.array(self.lookups.y), + separator=",", + threshold=len(self.lookups.y) + ) + return BuildAST( + expression="np.interp(%(value)s, %(x)s, %(y)s)" % arguments, + calls=arguments["value"].calls, + subscripts=arguments["value"].subscripts, + order=0) + + +class ReferenceBuilder(StructureBuilder): + """Builder for references to other variables.""" + def __init__(self, reference_str: ReferenceStructure, component: object): + super().__init__(None, component) + self.mapping_subscripts = {} + self.reference = reference_str.reference + self.subscripts = reference_str.subscripts + self.arguments = {} + self.section.imports.add("xarray") + + @property + def subscripts(self): + return self._subscripts + + @subscripts.setter + def subscripts(self, subscripts: SubscriptsReferenceStructure): + """Get subscript dictionary from reference""" + self._subscripts = self.section.subscripts.make_coord_dict( + getattr(subscripts, "subscripts", {})) + + # get the subscripts after applying the mapping if necessary + for dim, coordinates in self._subscripts.items(): + if len(coordinates) > 1: + # we create the mapping only with those subscripts that are + # ranges as we need to ignore singular subscripts because + # that dimension is removed from final element + if dim not in self.def_subs and not dim.endswith("!"): + # the reference has a subscripts which is it not + # applied (!) and does not appear in the definition + # of the variable + for mapped in self.section.subscripts.mapping[dim]: + # check the mapped subscripts + # TODO update this and the parser to make it + # compatible with more complex mappings + if mapped in self.def_subs\ + and mapped not in self._subscripts: + # the mapped subscript appears in the definition + # and it is not already in the variable + self.mapping_subscripts[mapped] =\ + self.section.subscripts.subscripts[mapped] + break + else: + # the subscript is in the variable definition, + # do not change it + self.mapping_subscripts[dim] = coordinates + + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + if self.reference not in self.section.namespace.cleanspace: + # Manage references to subscripts (subscripts used as variables) + expression, subscripts =\ + self.section.subscripts.subscript2num[self.reference] + subscripts_out = self.section.subscripts.simplify_subscript_input( + subscripts)[1] + if subscripts: + self.section.imports.add("numpy") + # NUMPY: not need this if + expression = "xr.DataArray(%s, %s, %s)" % ( + expression, subscripts_out, list(subscripts)) + return BuildAST( + expression=expression, + calls={}, + subscripts=subscripts, + order=0) + + reference = self.section.namespace.cleanspace[self.reference] + + expression = reference + "()" + + if not self.subscripts: + return BuildAST( + expression=expression, + calls={reference: 1}, + subscripts={}, + order=0) + + original_subs = self.section.subscripts.make_coord_dict( + self.section.subscripts.elements[reference]) + + expression, final_subs = self._visit_subscripts( + expression, original_subs) + + return BuildAST( + expression=expression, + calls={reference: 1}, + subscripts=final_subs, + order=0) + + def _visit_subscripts(self, expression: str, original_subs: dict) -> tuple: + """ + Visit the subcripts of a reference to subset a subarray if neccessary + or apply mapping. + + Parameters + ---------- + expression: str + The expression of visiting the variable. + original_subs: dict + The original subscript dict of the variable. + + Returns + ------- + expression: str + The expression with the necessary operations. + mapping_subscirpts: dict + The final subscripts of the reference after applying mapping. + + """ + loc, rename, final_subs, reset_coords, to_float =\ + visit_loc(self.subscripts, original_subs) + + if loc is not None: + # NUMPY: expression += "[%s]" % ", ".join(loc) + expression += f".loc[{loc}]" + if to_float: + # NUMPY: Not neccessary + expression = "float(" + expression + ")" + elif reset_coords: + # NUMPY: Not neccessary + expression += ".reset_coords(drop=True)" + if rename: + # NUMPY: Not neccessary + expression += ".rename(%s)" % rename + + # NUMPY: This will not be necessary, we only need to return + # self.mapping_subscripts + if self.mapping_subscripts != final_subs: + subscripts_out = self.section.subscripts.simplify_subscript_input( + self.mapping_subscripts)[1] + expression = "xr.DataArray(%s.values, %s, %s)" % ( + expression, subscripts_out, list(self.mapping_subscripts) + ) + + return expression, self.mapping_subscripts + + +class NumericBuilder(StructureBuilder): + """Builder for numeric and nan values.""" + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + if np.isnan(self.value): + self.section.imports.add("numpy") + + return BuildAST( + expression="np.nan", + calls={}, + subscripts={}, + order=0) + else: + return BuildAST( + expression=repr(self.value), + calls={}, + subscripts={}, + order=0) + + +class ArrayBuilder(StructureBuilder): + """Builder for arrays.""" + def build(self, arguments: dict) -> BuildAST: + """ + Build method. + + Parameters + ---------- + arguments: dict + The dictionary of builded arguments. + + Returns + ------- + built_ast: BuildAST + The built object. + + """ + self.value = np.array2string( + self.value.reshape(compute_shape(self.def_subs)), + separator=",", + threshold=np.prod(self.value.shape) + ) + self.component.type = "Constant" + self.component.subtype = "Normal" + + final_subs, subscripts_out =\ + self.section.subscripts.simplify_subscript_input( + self.def_subs, self.element.subscripts) + + return BuildAST( + expression="xr.DataArray(%s, %s, %s)" % ( + self.value, subscripts_out, list(final_subs)), + calls={}, + subscripts=final_subs, + order=0) + + +def merge_dependencies(*dependencies: dict, inplace: bool = False) -> dict: + """ + Merge two dependencies dicts of an element. + + Parameters + ---------- + dependencies: dict + The dictionaries of dependencies to merge. + + inplace: bool (optional) + If True the final dependencies dict will be updated in the first + dependencies argument, mutating it. Default is False. + + Returns + ------- + current: dict + The final dependencies dict. + + """ + current = dependencies[0] + if inplace: + current = dependencies[0] + else: + current = dependencies[0].copy() + for new in dependencies[1:]: + if not current: + current.update(new) + elif new: + # regular element + current_set, new_set = set(current), set(new) + for dep in current_set.intersection(new_set): + # if dependency is in both sum the number of calls + if dep.startswith("__"): + # if it is special (__lookup__, __external__) continue + continue + else: + current[dep] += new[dep] + for dep in new_set.difference(current_set): + # if dependency is only in new copy it + current[dep] = new[dep] + + return current + + +def visit_loc(current_subs: dict, original_subs: dict, + keep_shape: bool = False) -> tuple: + """ + Compares the original subscripts and the current subscripts and + returns subindexing information if needed. + + Parameters + ---------- + current_subs: dict + The dictionary of the subscripts that are used in the variable. + + original_subs: dict + The dictionary of the original subscripts of the variable. + + keep_shape: bool (optional) + If True will keep the number of dimensions of the original element + and return only loc. Default is False. + + Returns + ------- + loc: list of str or None + List of the subscripting in each dimensions. If all are full (":"), + None is rerned wich means that array indexing is not needed. + + rename: dict + Dictionary of the dimensions to rename. + + final_subs: dict + Dictionary of the final subscripts of the variable. + + reset_coords: bool + Boolean indicating if the coords need to be reseted. + + to_float: bool + Boolean indicating if the variable should be converted to a float. + + """ + final_subs, rename, loc, reset_coords, to_float = {}, {}, [], False, True + subscripts_zipped = zip(current_subs.items(), original_subs.items()) + for (dim, coord), (orig_dim, orig_coord) in subscripts_zipped: + if len(coord) == 1: + # subset a 1 dimension value + # NUMPY: subset value [:, N, :, :] + if keep_shape: + # NUMPY: not necessary + loc.append(f"[{repr(coord[0])}]") + else: + loc.append(repr(coord[0])) + reset_coords = True + elif len(coord) < len(orig_coord): + # subset a subrange + # NUMPY: subset value [:, :, np.array([1, 0]), :] + # NUMPY: as order may change we need to check if + # dim != orig_dim + # NUMPY: use also ranges [:, :, 2:5, :] when possible + if dim.endswith("!"): + loc.append("_subscript_dict['%s']" % dim[:-1]) + else: + if dim != orig_dim: + loc.append("_subscript_dict['%s']" % dim) + else: + # workaround for locs from external objects merge + loc.append(repr(coord)) + final_subs[dim] = coord + to_float = False + else: + # do nothing + # NUMPY: same, we can remove float = False + loc.append(":") + final_subs[dim] = coord + to_float = False + + if dim != orig_dim and len(coord) != 1: + # NUMPY: check order of dimensions, make all subranges work + # with the same dimensions? + # NUMPY: this could be solved in the previous if/then/else + rename[orig_dim] = dim + + if all(dim == ":" for dim in loc): + # if all are ":" then no need to loc + loc = None + else: + loc = ", ".join(loc) + + if keep_shape: + return loc + + # convert to float if also coords are reseted (input is an array) + to_float = to_float and reset_coords + + # NUMPY: save and return only loc, the other are not needed + return loc, rename, final_subs, reset_coords, to_float + + +class ASTVisitor: + """ + ASTVisitor allows visiting the Abstract Synatx Tree of a component + returning the Python object and generating the neccessary objects. + + Parameters + ---------- + component: ComponentBuilder + The component builder to build. + + """ + _builders = { + InitialStructure: InitialBuilder, + IntegStructure: IntegBuilder, + DelayStructure: lambda x, y: DelayBuilder("Delay", x, y), + DelayNStructure: lambda x, y: DelayBuilder("DelayN", x, y), + DelayFixedStructure: DelayFixedBuilder, + SmoothStructure: SmoothBuilder, + SmoothNStructure: SmoothBuilder, + TrendStructure: TrendBuilder, + ForecastStructure: ForecastBuilder, + SampleIfTrueStructure: SampleIfTrueBuilder, + GetConstantsStructure: ExtConstantBuilder, + GetDataStructure: ExtDataBuilder, + GetLookupsStructure: ExtLookupBuilder, + LookupsStructure: LookupsBuilder, + InlineLookupsStructure: InlineLookupsBuilder, + DataStructure: TabDataBuilder, + ReferenceStructure: ReferenceBuilder, + CallStructure: CallBuilder, + GameStructure: GameBuilder, + LogicStructure: OperationBuilder, + ArithmeticStructure: OperationBuilder, + int: NumericBuilder, + float: NumericBuilder, + np.ndarray: ArrayBuilder, + } + + def __init__(self, component: object): + # component typing should be ComponentBuilder, but importing it + # for typing would create a circular dependency :S + self.ast = component.ast + self.subscripts = component.subscripts_dict + self.component = component + + def visit(self) -> Union[None, BuildAST]: + """ + Visit the Abstract Syntax Tree of the component. + + Returns + ------- + visit_out: BuildAST or None + The BuildAST object resulting from visiting the AST. If the + component content has been added to an existing object + using the 'add' method it will return None. + + """ + visit_out = self._visit(self.ast) + + if not visit_out: + # external objects that are declared with other expression + return None + + if not visit_out.calls and self.component.type == "Auxiliary": + self.component.type = "Constant" + self.component.subtype = "Normal" + + # include dependencies of the current component in the element + merge_dependencies( + self.component.element.dependencies, + visit_out.calls, + inplace=True) + + if not visit_out.subscripts: + # expression is a float + return visit_out + + # NUMPY not needed + # get subscript in elements as name of the ranges may change + subscripts_in_element = { + dim: coords + for dim, coords + in zip(self.component.element.subscripts, self.subscripts.values()) + } + + reshape = ( + (visit_out.subscripts != self.subscripts + or list(visit_out.subscripts) != list(self.subscripts)) + and + (visit_out.subscripts != subscripts_in_element + or list(visit_out.subscripts) != list(subscripts_in_element)) + ) + + if reshape: + # NUMPY: in this case we need to tile along dims if neccessary + # or reorder the dimensions + visit_out.reshape( + self.component.section.subscripts, self.subscripts, True) + + return visit_out + + def _visit(self, ast_object: AbstractSyntax) -> AbstractSyntax: + """ + Visit one Builder and its arguments. + """ + builder = self._builders[type(ast_object)](ast_object, self.component) + arguments = { + name: self._visit(value) + for name, value in builder.arguments.items() + } + return builder.build(arguments) diff --git a/pysd/builders/python/python_functions.py b/pysd/builders/python/python_functions.py new file mode 100644 index 00000000..e8560205 --- /dev/null +++ b/pysd/builders/python/python_functions.py @@ -0,0 +1,99 @@ + +# functions that can be diretcly applied over an array +functionspace = { + # directly build functions without dependencies + "elmcount": ("len(%(0)s)", None), + + # directly build numpy based functions + "abs": ("np.abs(%(0)s)", ("numpy",)), + "min": ("np.minimum(%(0)s, %(1)s)", ("numpy",)), + "max": ("np.maximum(%(0)s, %(1)s)", ("numpy",)), + "exp": ("np.exp(%(0)s)", ("numpy",)), + "sin": ("np.sin(%(0)s)", ("numpy",)), + "cos": ("np.cos(%(0)s)", ("numpy",)), + "tan": ("np.tan(%(0)s)", ("numpy",)), + "arcsin": ("np.arcsin(%(0)s)", ("numpy",)), + "arccos": ("np.arccos(%(0)s)", ("numpy",)), + "arctan": ("np.arctan(%(0)s)", ("numpy",)), + "sinh": ("np.sinh(%(0)s)", ("numpy",)), + "cosh": ("np.cosh(%(0)s)", ("numpy",)), + "tanh": ("np.tanh(%(0)s)", ("numpy",)), + "sqrt": ("np.sqrt(%(0)s)", ("numpy",)), + "ln": ("np.log(%(0)s)", ("numpy",)), + "log": ("(np.log(%(0)s)/np.log(%(1)s))", ("numpy",)), + # NUMPY: "invert_matrix": ("np.linalg.inv(%(0)s)", ("numpy",)), + + # vector functions with axis to apply over + # NUMPY: + # "prod": "np.prod(%(0)s, axis=%(axis)s)", ("numpy",)), + # "sum": "np.sum(%(0)s, axis=%(axis)s)", ("numpy",)), + # "vmax": "np.max(%(0)s, axis=%(axis)s)", ("numpy", )), + # "vmin": "np.min(%(0)s, axis=%(axis)s)", ("numpy",)) + "prod": ("prod(%(0)s, dim=%(axis)s)", ("functions", "prod")), + "sum": ("sum(%(0)s, dim=%(axis)s)", ("functions", "sum")), + "vmax": ("vmax(%(0)s, dim=%(axis)s)", ("functions", "vmax")), + "vmin": ("vmin(%(0)s, dim=%(axis)s)", ("functions", "vmin")), + + # functions defined in pysd.py_bakcend.functions + "active_initial": ( + "active_initial(__data[\"time\"].stage, lambda: %(0)s, %(1)s)", + ("functions", "active_initial")), + "if_then_else": ( + "if_then_else(%(0)s, lambda: %(1)s, lambda: %(2)s)", + ("functions", "if_then_else")), + "integer": ( + "integer(%(0)s)", + ("functions", "integer")), + "invert_matrix": ( # NUMPY: remove + "invert_matrix(%(0)s)", + ("functions", "invert_matrix")), # NUMPY: remove + "modulo": ( + "modulo(%(0)s, %(1)s)", + ("functions", "modulo")), + "pulse": ( + "pulse(__data['time'], %(0)s, width=%(1)s)", + ("functions", "pulse")), + "Xpulse": ( + "pulse(__data['time'], %(0)s, magnitude=%(1)s)", + ("functions", "pulse")), + "pulse_train": ( + "pulse(__data['time'], %(0)s, repeat_time=%(1)s, width=%(2)s, "\ + "end=%(3)s)", + ("functions", "pulse")), + "Xpulse_train": ( + "pulse(__data['time'], %(0)s, repeat_time=%(1)s, magnitude=%(2)s)", + ("functions", "pulse")), + "quantum": ( + "quantum(%(0)s, %(1)s)", + ("functions", "quantum")), + "Xramp": ( + "ramp(__data['time'], %(0)s, %(1)s)", + ("functions", "ramp")), + "ramp": ( + "ramp(__data['time'], %(0)s, %(1)s, %(2)s)", + ("functions", "ramp")), + "step": ( + "step(__data['time'], %(0)s, %(1)s)", + ("functions", "step")), + "xidz": ( + "xidz(%(0)s, %(1)s, %(2)s)", + ("functions", "xidz")), + "zidz": ( + "zidz(%(0)s, %(1)s)", + ("functions", "zidz")), + + # random functions must have the shape of the component subscripts + # most of them are shifted, scaled and truncated + # TODO: it is difficult to find same parametrization in Python, + # maybe build a new model + "random_0_1": ( + "np.random.uniform(0, 1, size=%(size)s)", + ("numpy",)), + "random_uniform": ( + "np.random.uniform(%(0)s, %(1)s, size=%(size)s)", + ("numpy",)), + "random_normal": ( + "stats.truncnorm.rvs(%(0)s, %(1)s, loc=%(2)s, scale=%(3)s," + " size=%(size)s)", + ("scipy", "stats")), +} diff --git a/pysd/builders/python/python_model_builder.py b/pysd/builders/python/python_model_builder.py new file mode 100644 index 00000000..1f6e379f --- /dev/null +++ b/pysd/builders/python/python_model_builder.py @@ -0,0 +1,804 @@ +""" +The ModelBuilder class allows converting the AbstractModel into a +PySD model writing the Python code in files that can be loaded later +with PySD Model class. Each Abstract level has its own Builder. However, +the user is only required to create a ModelBuilder object using the +AbstractModel and call the `build_model` method. +""" +import textwrap +import black +import json +from pathlib import Path +from typing import Union + +from pysd.translators.structures.abstract_model import\ + AbstractComponent, AbstractElement, AbstractModel, AbstractSection + +from . import python_expressions_builder as vs +from .namespace import NamespaceManager +from .subscripts import SubscriptManager +from .imports import ImportsManager +from pysd._version import __version__ + + +class ModelBuilder: + """ + ModelBuilder allows building a PySD Python model from the + Abstract Model. + + Parameters + ---------- + abstract_model: AbstractModel + The abstract model to build. + + """ + + def __init__(self, abstract_model: AbstractModel): + self.__dict__ = abstract_model.__dict__.copy() + # load sections + self.sections = [ + SectionBuilder(section) + for section in abstract_model.sections + ] + # create the macrospace (namespace of macros) + self.macrospace = { + section.name: section for section in self.sections[1:]} + + def build_model(self) -> Path: + """ + Build the Python model in a file callled as the orginal model + but with '.py' suffix. + + Returns + ------- + path: pathlib.Path + The path to the new PySD model. + + """ + for section in self.sections: + # add macrospace information to each section and build it + section.macrospace = self.macrospace + section.build_section() + + # return the path to the main file + return self.sections[0].path + + +class SectionBuilder: + """ + SectionBuilder allows building a section of the PySD model. Each + section will be a file unless the model has been setted to be + split in modules. + + Parameters + ---------- + abstract_section: AbstractSection + The abstract section to build. + + """ + def __init__(self, abstract_section: AbstractSection): + self.__dict__ = abstract_section.__dict__.copy() + self.root = self.path.parent # the folder where the model is + self.model_name = self.path.with_suffix("").name # name of the model + # Create subscript manager object with subscripts_dict + self.subscripts = SubscriptManager( + abstract_section.subscripts, self.root) + # Load the elements in the section + self.elements = [ + ElementBuilder(element, self) + for element in abstract_section.elements + ] + # Create the namespace of the section + self.namespace = NamespaceManager(self.params) + # Create an imports manager + self.imports = ImportsManager() + # Create macrospace (namespace of macros) + self.macrospace = {} + # Create parameters dict necessary in macros + self.params = { + key: self.namespace.namespace[key] + for key in self.params + } + + def build_section(self) -> None: + """ + Build the Python section in a file callled as the orginal model + if the section is main or in a file called as the macro name + if the section is a macro. + """ + # Firts iteration over elements to recover their information + for element in self.elements: + # Add element to namespace + self.namespace.add_to_namespace(element.name) + identifier = self.namespace.namespace[element.name] + element.identifier = identifier + # Add element subscripts information to the subscript manager + self.subscripts.elements[identifier] = element.subscripts + + # Build elements + for element in self.elements: + element.build_element() + + if self.split: + # Build modular section + self._build_modular(self.views_dict) + else: + # Build one-file section + self._build() + + def _process_views_tree(self, view_name: str, + view_content: Union[dict, set], + wdir: Path) -> dict: + """ + Creates a directory tree based on the elements_per_view dictionary. + If it's the final view, it creates a file, if not, it creates a folder. + """ + if isinstance(view_content, set): + # Will become a module + # Convert subview elements names to Python names + view_content = { + self.namespace.cleanspace[var] for var in view_content + } + # Get subview elements + subview_elems = [ + element for element in self.elements_remaining + if element.identifier in view_content + ] + # Remove elements from remaining ones + [ + self.elements_remaining.remove(element) + for element in subview_elems + ] + # Build the module + self._build_separate_module(subview_elems, view_name, wdir) + return sorted(view_content) + else: + # The current view has subviews + wdir = wdir.joinpath(view_name) + wdir.mkdir(exist_ok=True) + return { + subview_name: + self._process_views_tree(subview_name, subview_content, wdir) + for subview_name, subview_content in view_content.items() + } + + def _build_modular(self, elements_per_view: dict) -> None: + """ Build modular section """ + self.elements_remaining = self.elements.copy() + elements_per_view = self._process_views_tree( + "modules_" + self.model_name, elements_per_view, self.root) + # Building main file using the build function + self._build_main_module(self.elements_remaining) + + # Build subscripts dir and moduler .json files + for file, values in { + "modules_%s/_modules": elements_per_view, + "_subscripts_%s": self.subscripts.subscripts}.items(): + + with self.root.joinpath( + file % self.model_name).with_suffix( + ".json").open("w") as outfile: + json.dump(values, outfile, indent=4, sort_keys=True) + + def _build_separate_module(self, elements: list, module_name: str, + module_dir: str) -> None: + """ + Constructs and writes the Python representation of a specific model + module, when the split_views=True in the read_vensim function. + + Parameters + ---------- + elements: list + Elements belonging to the module module_name. + + module_name: str + Name of the module + + module_dir: str + Path of the directory where module files will be stored. + + Returns + ------- + None + + """ + text = textwrap.dedent(''' + """ + Module %(module_name)s + Translated using PySD version %(version)s + """ + ''' % { + "module_name": module_name, + "version": __version__, + }) + funcs = self._generate_functions(elements) + text += funcs + text = black.format_file_contents( + text, fast=True, mode=black.FileMode()) + + outfile_name = module_dir.joinpath(module_name + ".py") + + with outfile_name.open("w", encoding="UTF-8") as out: + out.write(text) + + def _build_main_module(self, elements: list) -> None: + """ + Constructs and writes the Python representation of the main model + module, when the split_views=True in the read_vensim function. + + Parameters + ---------- + elements: list + Elements belonging to the main module. Ideally, there should + only be the initial_time, final_time, saveper and time_step, + functions, though there might be others in some situations. + Each element is a dictionary, with the various components + needed to assemble a model component in Python syntax. This + will contain multiple entries for elements that have multiple + definitions in the original file, and which need to be combined. + + Returns + ------- + None + + """ + # separating between control variables and rest of variables + control_vars, funcs = self._build_variables(elements) + + self.imports.add("utils", "load_model_data") + self.imports.add("utils", "load_modules") + + # import of needed functions and packages + text = self.imports.get_header(self.path.name) + + # import subscript dict from json file + text += textwrap.dedent(""" + __pysd_version__ = '%(version)s' + + __data = { + 'scope': None, + 'time': lambda: 0 + } + + _root = Path(__file__).parent + %(params)s + _subscript_dict, _modules = load_model_data( + _root, "%(model_name)s") + + component = Component() + """ % { + "params": f"\n _params = {self.params}\n" + if self.params else "", + "model_name": self.model_name, + "version": __version__ + }) + + text += self._get_control_vars(control_vars) + + text += textwrap.dedent(""" + # load modules from modules_%(model_name)s directory + exec(load_modules("modules_%(model_name)s", _modules, _root, [])) + + """ % { + "model_name": self.model_name, + }) + + text += funcs + text = black.format_file_contents( + text, fast=True, mode=black.FileMode()) + + with self.path.open("w", encoding="UTF-8") as out: + out.write(text) + + def _build(self) -> None: + """ + Constructs and writes the Python representation of a section. + + Returns + ------- + None + + """ + control_vars, funcs = self._build_variables(self.elements) + + text = self.imports.get_header(self.path.name) + indent = "\n " + # Generate params dict for macro parameters + params = f"{indent}_params = {self.params}\n"\ + if self.params else "" + # Generate subscripts dir + subs = f"{indent}_subscript_dict = {self.subscripts.subscripts}"\ + if self.subscripts.subscripts else "" + + text += textwrap.dedent(""" + __pysd_version__ = '%(version)s' + + __data = { + 'scope': None, + 'time': lambda: 0 + } + + _root = Path(__file__).parent + %(params)s + %(subscript_dict)s + + component = Component() + """ % { + "subscript_dict": subs, + "params": params, + "version": __version__, + }) + + text += self._get_control_vars(control_vars) + funcs + + text = black.format_file_contents( + text, fast=True, mode=black.FileMode()) + + with self.path.open("w", encoding="UTF-8") as out: + out.write(text) + + def _build_variables(self, elements: dict) -> tuple: + """ + Build model variables (functions) and separate then in control + variables and regular variables. + + Returns + ------- + control_vars, regular_vars: tuple, str + control_vars is a tuple of length 2. First element is the + dictionary of original control vars. Second is the string to + add the control variables' functions. regular_vars is the + string to add the regular variables' functions. + + """ + # returns of the control variables + control_vars_dict = { + "initial_time": "__data['time'].initial_time()", + "final_time": "__data['time'].final_time()", + "time_step": "__data['time'].time_step()", + "saveper": "__data['time'].saveper()" + } + regular_vars = [] + control_vars = [] + + for element in elements: + if element.identifier in control_vars_dict: + # change the return expression in the element and update + # the dict with the original expression + control_vars_dict[element.identifier], element.expression =\ + element.expression, control_vars_dict[element.identifier] + control_vars.append(element) + else: + regular_vars.append(element) + + if len(control_vars) == 0: + # macro objects, no control variables + control_vars_dict = "" + else: + control_vars_dict = """ + _control_vars = { + "initial_time": lambda: %(initial_time)s, + "final_time": lambda: %(final_time)s, + "time_step": lambda: %(time_step)s, + "saveper": lambda: %(saveper)s + } + """ % control_vars_dict + + return (control_vars_dict, + self._generate_functions(control_vars)),\ + self._generate_functions(regular_vars) + + def _generate_functions(self, elements: dict) -> str: + """ + Builds all model elements as functions in string format. + NOTE: this function calls the build_element function, which + updates the import_modules. + Therefore, it needs to be executed before the method + _generate_automatic_imports. + + Parameters + ---------- + elements: dict + Each element is a dictionary, with the various components + needed to assemble a model component in Python syntax. This + will contain multiple entries for elements that have multiple + definitions in the original file, and which need to be combined. + + Returns + ------- + funcs: str + String containing all formated model functions + + """ + return "\n".join( + [element._build_element_out() for element in elements] + ) + + def _get_control_vars(self, control_vars: str) -> str: + """ + Create the section of control variables + + Parameters + ---------- + control_vars: str + Functions to define control variables. + + Returns + ------- + text: str + Control variables section and header of model variables section. + + """ + text = textwrap.dedent(""" + ####################################################################### + # CONTROL VARIABLES # + ####################################################################### + %(control_vars_dict)s + def _init_outer_references(data): + for key in data: + __data[key] = data[key] + + + @component.add(name="Time") + def time(): + ''' + Current time of the model. + ''' + return __data['time']() + + """ % {"control_vars_dict": control_vars[0]}) + + text += control_vars[1] + + text += textwrap.dedent(""" + ####################################################################### + # MODEL VARIABLES # + ####################################################################### + """) + + return text + + +class ElementBuilder: + """ + ElementBuilder allows building an element of the PySD model. + + Parameters + ---------- + abstract_element: AbstractElement + The abstract element to build. + section: SectionBuilder + The section where the element is defined. Necessary to give the + acces to the subscripts and namespace. + + """ + def __init__(self, abstract_element: AbstractElement, + section: SectionBuilder): + self.__dict__ = abstract_element.__dict__.copy() + # Set element type and subtype to None + self.type = None + self.subtype = None + # Get the arguments of the element + self.arguments = getattr(self.components[0], "arguments", "") + # Load the components of the element + self.components = [ + ComponentBuilder(component, self, section) + for component in abstract_element.components + ] + self.section = section + # Get the subscripts of the element after merging all the components + self.subscripts = section.subscripts.make_merge_list( + [component.subscripts[0] for component in self.components]) + # Get the subscript dictionary of the element + self.subs_dict = section.subscripts.make_coord_dict(self.subscripts) + # Dictionaries to save dependencies and objects related to the element + self.dependencies = {} + self.other_dependencies = {} + self.objects = {} + + def build_element(self) -> None: + """ + Build the element. Returns the string to include in the section which + will be a decorated function definition and possible objects. + """ + # TODO think better how to build the components at once to build + # in one declaration the external objects + # TODO include some kind of magic vectorization to identify patterns + # that can be easily vecorized (GET, expressions, Stocks...) + + # Build the components of the element + [component.build_component() for component in self.components] + expressions = [] + for component in self.components: + expr, subs, except_subscripts = component.get() + if expr is None: + # The expr is None when the component has been "added" + # to an existing object using the add method + continue + if isinstance(subs, list): + # Get the list of locs for the component + # Subscripts dict will be a list when the component is + # translated to an object that groups may components + # via 'add' method. + loc = [vs.visit_loc(subsi, self.subs_dict, True) + for subsi in subs] + else: + # Get the loc of the component + loc = vs.visit_loc(subs, self.subs_dict, True) + + # Get the locs of the :EXCLUDE: parameters if any + exc_loc = [ + vs.visit_loc(subs_e, self.subs_dict, True) + for subs_e in except_subscripts + ] + expressions.append({ + "expr": expr, + "subs": subs, + "loc": loc, + "loc_except": exc_loc + }) + + if len(expressions) > 1: + # NUMPY: xrmerge would be sustitute by a multiple line definition + # e.g.: + # value = np.empty((len(dim1), len(dim2))) + # value[:, 0] = expression1 + # value[:, 1] = expression2 + # return value + # This allows reference to the same variable + # from: VAR[A] = 5; VAR[B] = 2*VAR[A] + # to: value[0] = 5; value[1] = 2*value[0] + self.section.imports.add("numpy") + self.pre_expression =\ + "value = xr.DataArray(np.nan, {%s}, %s)\n" % ( + ", ".join("'%(dim)s': _subscript_dict['%(dim)s']" % + {"dim": subs} for subs in self.subscripts), + self.subscripts) + for expression in expressions: + # Generate the pre_expression, operations to compute in + # the body of the function + if expression["expr"].subscripts: + # Get the values + # NUMPY not necessary + expression["expr"].lower_order(-1) + expression["expr"].expression += ".values" + if expression["loc_except"]: + # There is an excep in the definition of the component + self.pre_expression += self._manage_except(expression) + elif isinstance(expression["subs"], list): + # There are mixed definitions which include multicomponent + # object + self.pre_expression += self._manage_multi_def(expression) + else: + # Regular loc for a component + self.pre_expression +=\ + "value.loc[%(loc)s] = %(expr)s\n" % expression + + # Return value + self.expression = "value" + else: + self.pre_expression = "" + # NUMPY: reshape to the final shape if needed + # expressions[0]["expr"].reshape(self.section.subscripts, {}) + if not expressions[0]["expr"].subscripts and self.subscripts: + # Updimension the return value to an array + self.expression = "xr.DataArray(%s, %s, %s)\n" % ( + expressions[0]["expr"], + self.section.subscripts.simplify_subscript_input( + self.subs_dict)[1], + list(self.subs_dict) + ) + else: + # Return the expression + self.expression = expressions[0]["expr"] + + # Merge the types of the components (well defined element should + # have only one type and subtype) + self.type = ", ".join( + set(component.type for component in self.components) + ) + self.subtype = ", ".join( + set(component.subtype for component in self.components) + ) + + def _manage_multi_def(self, expression: dict) -> str: + """ + Manage multiline definitions when some of them (not all) are + merged to one object. + """ + final_expr = "def_subs = xr.zeros_like(value, dtype=bool)\n" + for loc in expression["loc"]: + # coordinates of the object + final_expr += f"def_subs.loc[{loc}] = True\n" + + # replace the values matching the coordinates + return final_expr + "value.values[def_subs.values] = "\ + "%(expr)s[def_subs.values]\n" % expression + + def _manage_except(self, expression: dict) -> str: + """ + Manage except declarations by not asigning its values. + """ + if expression["subs"] == self.subs_dict: + # Final subscripts are the same as the main subscripts + # of the component. Generate a True array like value + final_expr = "except_subs = xr.ones_like(value, dtype=bool)\n" + else: + # Final subscripts are greater than the main subscripts + # of the component. Generate a False array like value and + # set to True the subarray of the component coordinates + final_expr = "except_subs = xr.zeros_like(value, dtype=bool)\n"\ + "except_subs.loc[%(loc)s] = True\n" % expression + + for except_subs in expression["loc_except"]: + # We set to False the dimensions in the EXCEPT + final_expr += "except_subs.loc[%s] = False\n" % except_subs + + if expression["expr"].subscripts: + # Assign the values of an array + return final_expr + "value.values[except_subs.values] = "\ + "%(expr)s[except_subs.values]\n" % expression + else: + # Assign the values of a float + return final_expr + "value.values[except_subs.values] = "\ + "%(expr)s\n" % expression + + def _build_element_out(self) -> str: + """ + Returns a string that has processed a single element dictionary. + + Returns + ------- + func: str + The function to write in the model file. + + """ + # Contents of the function (body + return) + contents = self.pre_expression + "return %s" % self.expression + + # Get the objects to create as string + objects = "\n\n".join([ + value["expression"] % { + "final_subs": + self.section.subscripts.simplify_subscript_input( + value.get("final_subs", {}))[1] + } # Replace the final subs in the objects that merge + # several components + for value in self.objects.values() + if value["expression"] is not None + ]) + + # Format the limits to get them as a string + self.limits = self._format_limits(self.limits) + + # Update arguments with final subs to alllow passing arguments + # with subscripts to the lookups + if self.arguments == 'x': + self.arguments = 'x, final_subs=None' + + # Define variable metadata for the @component decorator + self.name = repr(self.name) + meta_data = ["name=%(name)s"] + + # Include basic metadata (units, limits, dimensions) + if self.units: + meta_data.append("units=%(units)s") + self.units = repr(self.units) + if self.limits: + meta_data.append("limits=%(limits)s") + if self.subscripts: + self.section.imports.add("subs") + meta_data.append("subscripts=%(subscripts)s") + + # Include component type and subtype + meta_data.append("comp_type='%(type)s'") + meta_data.append("comp_subtype='%(subtype)s'") + + # Include dependencies + if self.dependencies: + meta_data.append("depends_on=%(dependencies)s") + if self.other_dependencies: + meta_data.append("other_deps=%(other_dependencies)s") + + # Get metadata decorator + self.meta_data = f"@component.add({', '.join(meta_data)})"\ + % self.__dict__ + + # Clean the documentation and add it to the beggining of contents + if self.documentation: + doc = self.documentation.replace("\\", "\n") + contents = f'"""\n{doc}\n"""\n'\ + + contents + + indent = 12 + + # Convert newline indicator and add expected level of indentation + self.contents = contents.replace("\n", "\n" + " " * (indent+4)) + self.objects = objects.replace("\n", "\n" + " " * indent) + + # Return the decorated function definition with the object declarations + return textwrap.dedent(''' + %(meta_data)s + def %(identifier)s(%(arguments)s): + %(contents)s + + + %(objects)s + ''' % self.__dict__) + + def _format_limits(self, limits: tuple) -> str: + """Format the limits of an element to print them properly""" + if limits == (None, None): + return None + + new_limits = [] + for value in limits: + value = repr(value) + if value == "nan" or value == "None": + # add numpy.nan to the values + self.section.imports.add("numpy") + new_limits.append("np.nan") + elif value.endswith("inf"): + # add numpy.inf to the values + self.section.imports.add("numpy") + new_limits.append(value.strip("inf") + "np.inf") + else: + # add numeric value + new_limits.append(value) + + if new_limits[0] == "np.nan" and new_limits[1] == "np.nan": + # if both are numpy.nan do not include limits + return None + + return "(" + ", ".join(new_limits) + ")" + + +class ComponentBuilder: + """ + ComponentBuilder allows building a component of the PySD model. + + Parameters + ---------- + abstract_component: AbstracComponent + The abstract component to build. + element: ElementBuilder + The element where the component is defined. Necessary to give the + acces to the merging subscripts and other components. + section: SectionBuilder + The section where the element is defined. Necessary to give the + acces to the subscripts and namespace. + + """ + def __init__(self, abstract_component: AbstractComponent, + element: ElementBuilder, section: SectionBuilder): + self.__dict__ = abstract_component.__dict__.copy() + self.element = element + self.section = section + if not hasattr(self, "keyword"): + self.keyword = None + + def build_component(self) -> None: + """ + Build model component parsing the Abstract Syntax Tree. + """ + self.subscripts_dict = self.section.subscripts.make_coord_dict( + self.subscripts[0]) + self.except_subscripts = [self.section.subscripts.make_coord_dict( + except_list) for except_list in self.subscripts[1]] + self.ast_build = vs.ASTVisitor(self).visit() + + def get(self) -> tuple: + """ + Get build component to build the element. + + Returns + ------- + ast_build: BuildAST + Parsed AbstractSyntaxTree. + subscript_dict: dict or list of dicts + The subscripts of the component. + except_subscripts: list of dicts + The subscripts to avoid. + + """ + return self.ast_build, self.subscripts_dict, self.except_subscripts diff --git a/pysd/builders/python/subscripts.py b/pysd/builders/python/subscripts.py new file mode 100644 index 00000000..70b92018 --- /dev/null +++ b/pysd/builders/python/subscripts.py @@ -0,0 +1,393 @@ +import warnings +from pathlib import Path +import numpy as np +from typing import List + +from pysd.translators.structures.abstract_model import AbstractSubscriptRange +from pysd.py_backend.external import ExtSubscript + + +class SubscriptManager: + """ + SubscriptManager object allows saving the subscripts included in the + Section, searching for elements or keys and simplifying them. + + Parameters + ---------- + abstrac_subscripts: list + List of the AbstractSubscriptRanges comming from the AbstractModel. + + _root: pathlib.Path + Path to the model file. Needed to read subscript ranges from + Excel files. + + """ + def __init__(self, abstract_subscripts: List[AbstractSubscriptRange], + _root: Path): + self._root = _root + self._copied = [] + self.mapping = {} + self.subscripts = abstract_subscripts + self.elements = {} + self.subranges = self._get_main_subscripts() + self.subscript2num = self._get_subscript2num() + + @property + def subscripts(self) -> dict: + return self._subscripts + + @subscripts.setter + def subscripts(self, abstract_subscripts: List[AbstractSubscriptRange]): + self._subscripts = {} + missing = [] + for sub in abstract_subscripts: + self.mapping[sub.name] = sub.mapping + if isinstance(sub.subscripts, list): + # regular definition of subscripts + self._subscripts[sub.name] = sub.subscripts + elif isinstance(sub.subscripts, str): + # copied subscripts, this will be always a subrange, + # then we need to prevent them of being saved as a main range + self._copied.append(sub.name) + self.mapping[sub.name].append(sub.subscripts) + if sub.subscripts in self._subscripts: + self._subscripts[sub.name] =\ + self._subscripts[sub.subscripts] + else: + missing.append(sub) + elif isinstance(sub.subscripts, dict): + # subscript from file + self._subscripts[sub.name] = ExtSubscript( + file_name=sub.subscripts["file"], + sheet=sub.subscripts["tab"], + firstcell=sub.subscripts["firstcell"], + lastcell=sub.subscripts["lastcell"], + prefix=sub.subscripts["prefix"], + root=self._root).subscript + else: + raise ValueError( + f"Invalid definition of subscript '{sub.name}':\n\t" + + str(sub.subscripts)) + + while missing: + # second loop for copied subscripts + sub = missing.pop() + self._subscripts[sub.name] =\ + self._subscripts[sub.subscripts] + + def _get_main_subscripts(self) -> dict: + """ + Reutrns a dictionary with the main ranges as keys and their + subranges as values. + """ + subscript_sets = { + name: set(subs) for name, subs in self.subscripts.items()} + + subranges = {} + for range, subs in subscript_sets.items(): + # current subscript range + subranges[range] = [] + for subrange, subs2 in subscript_sets.items(): + if range == subrange: + # pass current range + continue + elif subs == subs2: + # range is equal to the subrange, as Vensim does + # the main range will be the first one alphabetically + # make it case insensitive + range_l = range.replace(" ", "_").lower() + subrange_l = subrange.replace(" ", "_").lower() + if range_l < subrange_l and range not in self._copied: + subranges[range].append(subrange) + else: + # copied subscripts ranges or subscripts ranges + # that come later alphabetically + del subranges[range] + break + elif subs2.issubset(subs): + # subrange is a subset of range, append it to the list + subranges[range].append(subrange) + elif subs2.issuperset(subs): + # it exist a range that contents the elements of the range + del subranges[range] + break + + return subranges + + def _get_subscript2num(self) -> dict: + """ + Build a dictionary to return the numeric value or values of a + subscript or subscript range. + """ + s2n = {} + for range, subranges in self.subranges.items(): + # a main range is direct to return + s2n[range.replace(" ", "_").lower()] = ( + f"np.arange(1, len(_subscript_dict['{range}'])+1)", + {range: self.subscripts[range]} + ) + for i, sub in enumerate(self.subscripts[range], start=1): + # a subscript must return its numeric position + # in the main range + s2n[sub.replace(" ", "_").lower()] = (str(i), {}) + for subrange in subranges: + # subranges may return the position of each subscript + # in the main range + sub_index = [ + self.subscripts[range].index(sub)+1 + for sub in self.subscripts[subrange]] + + if np.all( + sub_index + == np.arange(sub_index[0], sub_index[0]+len(sub_index))): + # subrange definition can be simplified with a range + subsarray = f"np.arange({sub_index[0]}, "\ + f"len(_subscript_dict['{subrange}'])+{sub_index[0]})" + else: + # subrange definition cannot be simplified + subsarray = f"np.array({sub_index})" + + s2n[subrange.replace(" ", "_").lower()] = ( + subsarray, + {subrange: self.subscripts[subrange]} + ) + + return s2n + + def _find_subscript_name(self, element: str, avoid: List[str] = []) -> str: + """ + Given a member of a subscript family, return the first key of + which the member is within the value list. + + Parameters + ---------- + element: str + Subscript or subscriptrange name to find. + avoid: list (optional) + List of subscripts to avoid. Default is an empty list. + + Returns + ------- + name: str + The first key of which the member is within the value list + in the subscripts dictionary. + + Examples + -------- + >>> sm = SubscriptManager([], Path('')) + >>> sm._subscripts = { + ... 'Dim1': ['A', 'B', 'C'], + ... 'Dim2': ['A', 'B', 'C', 'D']} + >>> sm._find_subscript_name('D') + 'Dim2' + >>> sm._find_subscript_name('B') + 'Dim1' + >>> sm._find_subscript_name('B', avoid=['Dim1']) + 'Dim2' + + """ + for name, elements in self.subscripts.items(): + if element in elements and name not in avoid: + return name + + def make_coord_dict(self, subs: List[str]) -> dict: + """ + This is for assisting with the lookup of a particular element. + + Parameters + ---------- + subs: list of strings + Coordinates, either as names of dimensions, or positions within + a dimension. + + Returns + ------- + coordinates: dict + Coordinates needed to access the xarray quantities we are + interested in. + + Examples + -------- + >>> sm = SubscriptManager([], Path('')) + >>> sm._subscripts = { + ... 'Dim1': ['A', 'B', 'C'], + ... 'Dim2': ['A', 'B', 'C', 'D']} + >>> sm.make_coord_dict(['Dim1', 'D']) + {'Dim1': ['A', 'B', 'C'], 'Dim2': ['D']} + >>> sm.make_coord_dict(['A']) + {'Dim1': ['A']} + >>> sm.make_coord_dict(['A', 'B']) + {'Dim1': ['A'], 'Dim2': ['B']} + >>> sm.make_coord_dict(['A', 'Dim1']) + {'Dim2': ['A'], 'Dim1': ['A', 'B', 'C']} + + """ + sub_elems_list = [y for x in self.subscripts.values() for y in x] + coordinates = {} + for sub in subs: + if sub in sub_elems_list: + name = self._find_subscript_name( + sub, avoid=subs + list(coordinates)) + coordinates[name] = [sub] + else: + if sub.endswith("!"): + coordinates[sub] = self.subscripts[sub[:-1]] + else: + coordinates[sub] = self.subscripts[sub] + return coordinates + + def make_merge_list(self, subs_list: List[List[str]], + element: str = "") -> List[str]: + """ + This is for assisting when building xrmerge. From a list of subscript + lists returns the final subscript list after mergin. Necessary when + merging variables with subscripts comming from different definitions. + + Parameters + ---------- + subs_list: list of lists of strings + Coordinates, either as names of dimensions, or positions within + a dimension. + element: str (optional) + Element name, if given it will be printed with any error or + warning message. Default is "". + + Returns + ------- + dims: list + Final subscripts after merging. + + Examples + -------- + >>> sm = SubscriptManager([], Path('')) + >>> sm._subscripts = {"upper": ["A", "B"], "all": ["A", "B", "C"]} + >>> sm.make_merge_list([['A'], ['B']]) + ['upper'] + >>> sm.make_merge_list([['A'], ['B'], ['C']]) + ['all'] + >>> sm.make_merge_list([['upper'], ['C']]) + ['all'] + >>> sm.make_merge_list([['A'], ['C']]) + ['all'] + + """ + coords_set = [set() for i in range(len(subs_list[0]))] + coords_list = [ + self.make_coord_dict(subs) + for subs in subs_list + ] + + # update coords set + [[coords_set[i].update(coords[dim]) for i, dim in enumerate(coords)] + for coords in coords_list] + + dims = [None] * len(coords_set) + # create an array with the name of the subranges for all + # merging elements + dims_list = np.array([ + list(coords) for coords in coords_list]).transpose() + indexes = np.arange(len(dims)) + + for i, coord2 in enumerate(coords_set): + dims1 = [ + dim for dim in dims_list[i] + if dim is not None and set(self.subscripts[dim]) == coord2 + ] + if dims1: + # if the given coordinate already matches return it + dims[i] = dims1[0] + else: + # find a suitable coordinate + other_dims = dims_list[indexes != i] + for name, elements in self.subscripts.items(): + if coord2 == set(elements) and name not in other_dims: + dims[i] = name + break + + if not dims[i]: + # the dimension is incomplete use the smaller + # dimension that completes it + for name, elements in self.subscripts.items(): + if coord2.issubset(set(elements))\ + and name not in other_dims: + dims[i] = name + warnings.warn( + element + + "\nDimension given by subscripts:" + + "\n\t{}\nis incomplete ".format(coord2) + + "using {} instead.".format(name) + + "\nSubscript_dict:" + + "\n\t{}".format(self.subscripts) + ) + break + + if not dims[i]: + for name, elements in self.subscripts.items(): + if coord2 == set(elements): + j = 1 + while name + str(j) in self.subscripts.keys(): + j += 1 + self.subscripts[name + str(j)] = elements + dims[i] = name + str(j) + warnings.warn( + element + + "\nAdding new subscript range to" + + " subscript_dict:\n" + + name + str(j) + ": " + ', '.join(elements)) + break + + return dims + + def simplify_subscript_input(self, coords: dict, + merge_subs: List[str] = None) -> tuple: + """ + Simplifies the subscripts input to avoid printing the coordinates + list when the _subscript_dict can be used. Makes model code more + simple. + + Parameters + ---------- + coords: dict + Coordinates to write in the model file. + + merge_subs: list of strings or None (optional) + List of the final subscript range of the Python array after + merging with other objects. If None the merge_subs will be + taken from coords. Default is None. + + Returns + ------- + final_subs, coords: dict, str + Final subscripts and the equations to generate the coord + dicttionary in the model file. + + Examples + -------- + >>> sm = SubscriptManager([], Path('')) + >>> sm._subscripts = { + ... "dim": ["A", "B", "C"], + ... "dim2": ["A", "B", "C", "D"]} + >>> sm.simplify_subscript_input({"dim": ["A", "B", "C"]}) + ({"dim": ["A", "B", "C"]}, "{'dim': _subscript_dict['dim']}" + >>> sm.simplify_subscript_input({"dim": ["A", "B", "C"]}, ["dim2"]) + ({"dim2": ["A", "B", "C"]}, "{'dim2': _subscript_dict['dim']}" + >>> sm.simplify_subscript_input({"dim": ["A", "B"]}) + ({"dim": ["A", "B"]}, "{'dim': ['A', 'B']}" + + """ + if merge_subs is None: + merge_subs = list(coords) + + coordsp = [] + final_subs = {} + for ndim, (dim, coord) in zip(merge_subs, coords.items()): + # find dimensions can be retrieved from _subscript_dict + final_subs[ndim] = coord + if not dim.endswith("!") and coord == self.subscripts[dim]: + # use _subscript_dict + coordsp.append(f"'{ndim}': _subscript_dict['{dim}']") + else: + # write whole dict + coordsp.append(f"'{ndim}': {coord}") + + return final_subs, "{" + ", ".join(coordsp) + "}" diff --git a/pysd/cli/main.py b/pysd/cli/main.py index ec9f2302..29601b52 100644 --- a/pysd/cli/main.py +++ b/pysd/cli/main.py @@ -1,5 +1,6 @@ import sys import os +from pathlib import Path from csv import QUOTE_NONE from datetime import datetime @@ -7,6 +8,10 @@ from .parser import parser import pysd +from pysd.translators.vensim.vensim_utils import supported_extensions as\ + vensim_extensions +from pysd.translators.xmile.xmile_utils import supported_extensions as\ + xmile_extensions def main(args): @@ -69,7 +74,7 @@ def load(model_file, data_files, missing_values, split_views, **kwargs): split_views: bool (optional) If True, the sketch is parsed to detect model elements in each - model view, and then translate each view in a separate python + model view, and then translate each view in a separate Python file. Setting this argument to True is recommended for large models split in many different views. Default is False. @@ -85,13 +90,14 @@ def load(model_file, data_files, missing_values, split_views, **kwargs): pysd.model """ - if model_file.lower().endswith(".mdl"): + model_file_suffix = Path(model_file).suffix.lower() + if model_file_suffix in vensim_extensions: print("\nTranslating model file...\n") return pysd.read_vensim(model_file, initialize=False, data_files=data_files, missing_values=missing_values, split_views=split_views, **kwargs) - elif model_file.lower().endswith(".xmile"): + elif model_file_suffix in xmile_extensions: print("\nTranslating model file...\n") return pysd.read_xmile(model_file, initialize=False, data_files=data_files, diff --git a/pysd/cli/parser.py b/pysd/cli/parser.py index b748f402..2a59a662 100644 --- a/pysd/cli/parser.py +++ b/pysd/cli/parser.py @@ -8,6 +8,10 @@ from argparse import ArgumentParser, Action from pysd import __version__ +from pysd.translators.vensim.vensim_utils import supported_extensions as\ + vensim_extensions +from pysd.translators.xmile.xmile_utils import supported_extensions as\ + xmile_extensions docs = "https://pysd.readthedocs.io/en/master/command_line_usage.html" @@ -38,18 +42,17 @@ def check_model(string): Checks that model file ends with .py .mdl or .xmile and that exists. """ - if not string.lower().endswith('.mdl')\ - and not string.lower().endswith('.xmile')\ - and not string.endswith('.py'): + suffixes = [".py"] + vensim_extensions + xmile_extensions + if not any(string.lower().endswith(suffix) for suffix in suffixes): parser.error( - f'when parsing {string}' - '\nThe model file name must be Vensim (.mdl), Xmile (.xmile)' - ' or PySD (.py) model file...') + f"when parsing {string} \nThe model file name must be a Vensim" + f" ({', '.join(vensim_extensions)}), a Xmile " + f"({', '.join(xmile_extensions)}) or a PySD (.py) model file...") if not os.path.isfile(string): parser.error( - f'when parsing {string}' - '\nThe model file does not exist...') + f"when parsing {string}" + "\nThe model file does not exist...") return string diff --git a/pysd/py_backend/decorators.py b/pysd/py_backend/cache.py similarity index 53% rename from pysd/py_backend/decorators.py rename to pysd/py_backend/cache.py index f796dee1..25d76481 100644 --- a/pysd/py_backend/decorators.py +++ b/pysd/py_backend/cache.py @@ -4,37 +4,6 @@ """ from functools import wraps import inspect -import xarray as xr - - -def subs(dims, subcoords): - """ - This decorators returns the python object with the correct dimensions - xarray.DataArray. The algorithm is a simple version of utils.rearrange - """ - def decorator(function): - function.dims = dims - - @wraps(function) - def wrapper(*args): - data = function(*args) - coords = {dim: subcoords[dim] for dim in dims} - - if isinstance(data, xr.DataArray): - dacoords = {coord: list(data.coords[coord].values) - for coord in data.coords} - if data.dims == tuple(dims) and dacoords == coords: - # If the input data already has the output format - # return it. - return data - - # The coordinates are expanded or transposed - return xr.DataArray(0, coords, dims) + data - - return xr.DataArray(data, coords, dims) - - return wrapper - return decorator class Cache(object): @@ -48,7 +17,6 @@ def __init__(self): def __call__(self, func, *args): """ Decorator for caching """ - func.args = inspect.getfullargspec(func)[0] @wraps(func) def cached_func(*args): diff --git a/pysd/py_backend/components.py b/pysd/py_backend/components.py index 5eccf370..95ad8e17 100644 --- a/pysd/py_backend/components.py +++ b/pysd/py_backend/components.py @@ -2,13 +2,49 @@ Model components and time managing classes. """ +from warnings import warn import os import random +import inspect from importlib.machinery import SourceFileLoader +import numpy as np + from pysd._version import __version__ +class Component(object): + + def __init__(self): + self.namespace = {} + self.dependencies = {} + + def add(self, name, units=None, limits=(np.nan, np.nan), + subscripts=None, comp_type=None, comp_subtype=None, + depends_on={}, other_deps={}): + """ + This decorators allows assigning metadata to a function. + """ + def decorator(function): + function.name = name + function.units = units + function.limits = limits + function.subscripts = subscripts + function.type = comp_type + function.subtype = comp_subtype + function.args = inspect.getfullargspec(function)[0] + + # include component in namespace and dependencies + self.namespace[name] = function.__name__ + if function.__name__ != "time": + self.dependencies[function.__name__] = depends_on + self.dependencies.update(other_deps) + + return function + + return decorator + + class Components(object): """ Workaround class to let the user do: @@ -44,7 +80,7 @@ def _load(self, py_model_file): "\n\nNot able to import the model. " + "This may be because the model was compiled with an " + "earlier version of PySD, you can check on the top of " - + " the model file you are trying to load." + + "the model file you are trying to load." + "\nThe current version of PySd is :" + "\n\tPySD " + __version__ + "\n\n" + "Please translate again the model with the function" @@ -84,6 +120,8 @@ def _set_component(self, name, value): class Time(object): + rprec = 1e-10 # relative precission for final time and saving time + def __init__(self): self._time = None self.stage = None @@ -135,29 +173,67 @@ def in_bounds(self): True if time is smaller than final time. Otherwise, returns Fase. """ - return self._time < self.final_time() + return self._time + self.time_step()*self.rprec < self.final_time() def in_return(self): """ Check if current time should be returned """ + prec = self.time_step() * self.rprec + if self.return_timestamps is not None: - return self._time in self.return_timestamps + # this allows managing float precission error + if self.next_return is None: + return False + if np.isclose(self._time, self.next_return, prec): + self._update_next_return() + return True + else: + while self.next_return is not None\ + and self._time > self.next_return: + warn( + f"The returning time stamp '{self.next_return}' " + "seems to not be a multiple of the time step. " + "This value will not be saved in the output. " + "Please, modify the returning timestamps or the " + "integration time step to avoid this." + ) + self._update_next_return() + return False time_delay = self._time - self._initial_time save_per = self.saveper() - prec = self.time_step() * 1e-10 return time_delay % save_per < prec or -time_delay % save_per < prec + def round(self): + """ Return rounded time to outputs to avoid float precission error""" + return np.round( + self._time, + -int(np.log10(self.time_step()*self.rprec))) + def add_return_timestamps(self, return_timestamps): """ Add return timestamps """ - if return_timestamps is None or hasattr(return_timestamps, '__len__'): - self.return_timestamps = return_timestamps + if hasattr(return_timestamps, '__len__')\ + and len(return_timestamps) > 0: + self.return_timestamps = list(return_timestamps) + self.return_timestamps.sort(reverse=True) + self.next_return = self.return_timestamps.pop() + elif isinstance(return_timestamps, (float, int)): + self.next_return = return_timestamps + self.return_timestamps = [] else: - self.return_timestamps = [return_timestamps] + self.next_return = None + self.return_timestamps = None def update(self, value): """ Update current time value """ self._time = value + def _update_next_return(self): + """ Update the next_return value """ + if self.return_timestamps: + self.next_return = self.return_timestamps.pop() + else: + self.next_return = None + def reset(self): """ Reset time value to the initial """ self._time = self._initial_time diff --git a/pysd/py_backend/data.py b/pysd/py_backend/data.py index 4a69d6fa..468e5935 100644 --- a/pysd/py_backend/data.py +++ b/pysd/py_backend/data.py @@ -1,5 +1,6 @@ import warnings import re +import random from pathlib import Path import numpy as np @@ -50,6 +51,8 @@ def read_file(cls, file_name, encoding=None): indicate if the output file is transposed. """ + # in the most cases variables will be split per columns, then + # read the first row to have all the column names out = cls.read_line(file_name, encoding) if out is None: raise ValueError( @@ -59,10 +62,16 @@ def read_file(cls, file_name, encoding=None): transpose = False try: - [float(col) for col in out] - out = cls.read_row(file_name, encoding) + # if we fail converting columns to float then they are + # not numeric values, so current direction is okay + [float(col) for col in random.sample(out, min(3, len(out)))] + # we did not fail, read the first column to see if variables + # are split per rows + out = cls.read_col(file_name, encoding) transpose = True - [float(col) for col in out] + # if we still are able to transform values to float the + # file is not valid + [float(col) for col in random.sample(out, min(3, len(out)))] except ValueError: return out, transpose else: @@ -91,7 +100,7 @@ def read_line(cls, file_name, encoding=None): return None @classmethod - def read_row(cls, file_name, encoding=None): + def read_col(cls, file_name, encoding=None): """ Read the firts column and return a set of it. """ @@ -172,12 +181,38 @@ class Data(object): # as Data # def __init__(self, data, coords, interp="interpolate"): + def set_values(self, values): + """Set new values from user input""" + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + + if isinstance(values, pd.Series): + index = list(values.index) + index.sort() + self.data = self.data.expand_dims( + {'time': index}, axis=0).copy() + + for index, value in values.items(): + if isinstance(values.values[0], xr.DataArray): + self.data.loc[index].loc[value.coords] =\ + value + else: + self.data.loc[index] = value + else: + if isinstance(values, xr.DataArray): + self.data.loc[values.coords] = values.values + else: + if self.final_coords: + self.data.loc[:] = values + else: + self.data = values + def __call__(self, time): try: if time in self.data['time'].values: outdata = self.data.sel(time=time) elif self.interp == "raw": - return np.nan + return self.nan elif time > self.data['time'].values[-1]: warnings.warn( self.py_name + "\n" @@ -190,9 +225,9 @@ def __call__(self, time): outdata = self.data[0] elif self.interp == "interpolate": outdata = self.data.interp(time=time) - elif self.interp == 'look forward': + elif self.interp == 'look_forward': outdata = self.data.sel(time=time, method="backfill") - elif self.interp == 'hold backward': + elif self.interp == 'hold_backward': outdata = self.data.sel(time=time, method="pad") if self.is_float: @@ -201,29 +236,40 @@ def __call__(self, time): else: # Remove time coord from the DataArray return outdata.reset_coords('time', drop=True) - except Exception as err: + except (TypeError, KeyError): if self.data is None: raise ValueError( self.py_name + "\n" "Trying to interpolate data variable before loading" " the data...") - else: - # raise any other possible error - raise err + + # this except catch the errors when a data has been + # changed to a constant value by the user + return self.data + except Exception as err: + raise err class TabData(Data): """ - Data from tabular file tab/cls, it could be from Vensim output. + Data from tabular file tab/csv, it could be from Vensim output. """ def __init__(self, real_name, py_name, coords, interp="interpolate"): self.real_name = real_name self.py_name = py_name self.coords = coords - self.interp = interp + self.final_coords = coords + self.interp = interp.replace(" ", "_") if interp else None self.is_float = not bool(coords) self.data = None + if self.interp not in ["interpolate", "raw", + "look_forward", "hold_backward"]: + raise ValueError(self.py_name + "\n" + + "The interpolation method (interp) must be " + + "'raw', 'interpolate', " + + "'look_forward' or 'hold_backward'") + def load_data(self, file_names): """ Load data values from files. @@ -279,6 +325,7 @@ def _load_data(self, file_name): if not self.coords: # 0 dimensional data + self.nan = np.nan values = load_outputs(file_name, transpose, columns=columns) return xr.DataArray( values.iloc[:, 0].values, @@ -290,6 +337,7 @@ def _load_data(self, file_name): values = load_outputs(file_name, transpose, columns=columns) + self.nan = xr.DataArray(np.nan, self.coords, dims) out = xr.DataArray( np.nan, {'time': values.index.values, **self.coords}, diff --git a/pysd/py_backend/external.py b/pysd/py_backend/external.py index c54a815a..e4f7190f 100644 --- a/pysd/py_backend/external.py +++ b/pysd/py_backend/external.py @@ -6,13 +6,13 @@ import re import warnings -from pathlib import Path import pandas as pd # TODO move to openpyxl import numpy as np import xarray as xr from openpyxl import load_workbook from . import utils from .data import Data +from .lookups import Lookups class Excels(): @@ -68,7 +68,7 @@ class External(object): Attributes ---------- py_name: str - The python name of the object + The Python name of the object missing: str ("warning", "error", "ignore", "keep") What to do with missing values. If "warning" (default) shows a warning message and interpolates the values. @@ -180,7 +180,7 @@ def _get_data_from_file_opyxl(self, cellname): # key error if the cellrange doesn't exist in the file or sheet raise AttributeError( self.py_name + "\n" - + "The cell range name:\t {}\n".format(cellname) + + "The cell range name:\t'{}'\n".format(cellname) + "Doesn't exist in:\n" + self._file_sheet ) @@ -265,8 +265,8 @@ def _get_series_data(self, series_across, series_row_or_col, cell, size): self.py_name + "\n" + "Dimension given in:\n" + self._file_sheet - + "\tDimentime_missingsion name:" - + "\t{}\n".format(series_row_or_col) + + "\tDimension name:" + + "\t'{}'\n".format(series_row_or_col) + " is a table and not a vector" ) @@ -285,8 +285,8 @@ def _get_series_data(self, series_across, series_row_or_col, cell, size): self.py_name + "\n" + "Dimension and data given in:\n" + self._file_sheet - + "\tDimension name:\t{}\n".format(series_row_or_col) - + "\tData name:\t{}\n".format(cell) + + "\tDimension name:\t'{}'\n".format(series_row_or_col) + + "\tData name:\t'{}'\n".format(cell) + " don't have the same length in the 1st dimension" ) @@ -297,7 +297,7 @@ def _get_series_data(self, series_across, series_row_or_col, cell, size): self.py_name + "\n" + "Data given in:\n" + self._file_sheet - + "\tData name:\t{}\n".format(cell) + + "\tData name:\t'{}'\n".format(cell) + " has not the same size as the given coordinates" ) @@ -322,16 +322,11 @@ def _resolve_file(self, root): None """ - if self.file[0] == '?': + if str(self.file)[0] == '?': # TODO add an option to include indirect references raise ValueError( self.py_name + "\n" - + f"Indirect reference to file: {self.file}") - - if isinstance(root, str): # pragma: no cover - # backwards compatibility - # TODO: remove with PySD 3.0.0 - root = Path(root) + + f"Indirect reference to file: '{self.file}'") self.file = root.joinpath(self.file) @@ -380,7 +375,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Dimension given in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(series_across, self.x_row_or_col) + + "\t{}:\t'{}'\n".format(series_across, self.x_row_or_col) + " has length 0" ) @@ -398,7 +393,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Dimension given in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(series_across, self.x_row_or_col) + + "\t{}:\t'{}'\n".format(series_across, self.x_row_or_col) + " has length 0" ) if self.missing == "warning": @@ -406,7 +401,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Dimension value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(series_across, self.x_row_or_col) + + "\t{}:\t'{}'\n".format(series_across, self.x_row_or_col) + " the corresponding data value(s) to the " + "missing/non-valid value(s) will be ignored\n\n" ) @@ -415,7 +410,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Dimension value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(series_across, self.x_row_or_col) + + "\t{}:\t'{}'\n".format(series_across, self.x_row_or_col) ) # reorder data with increasing series @@ -428,7 +423,7 @@ def _initialize_data(self, element_type): raise ValueError(self.py_name + "\n" + "Dimension given in:\n" + self._file_sheet - + "\t{}:\t{}\n".format( + + "\t{}:\t'{}'\n".format( series_across, self.x_row_or_col) + " has repeated values") @@ -453,7 +448,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Data value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(cell_type, self.cell) + + "\t{}:\t'{}'\n".format(cell_type, self.cell) + interpolate_message + "\n\n" ) elif self.missing == "raise": @@ -461,7 +456,7 @@ def _initialize_data(self, element_type): self.py_name + "\n" + "Data value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(cell_type, self.cell) + + "\t{}:\t'{}'\n".format(cell_type, self.cell) ) # fill values if self.interp != "raw": @@ -551,9 +546,9 @@ def _interpolate_missing(self, x, xr, yr): y[i] = yr[-1] elif value <= xr[0]: y[i] = yr[0] - elif self.interp == 'look forward': + elif self.interp == 'look_forward': y[i] = yr[xr >= value][0] - elif self.interp == 'hold backward': + elif self.interp == 'hold_backward': y[i] = yr[xr <= value][-1] else: y[i] = np.interp(value, xr, yr) @@ -564,8 +559,8 @@ def _file_sheet(self): """ Returns file and sheet name in a string """ - return "\tFile name:\t{}\n".format(self.file)\ - + "\tSheet name:\t{}\n".format(self.sheet) + return "\tFile name:\t'{}'\n".format(self.file)\ + + "\tSheet name:\t'{}'\n".format(self.sheet) @staticmethod def _col_to_num(col): @@ -646,10 +641,10 @@ def _reshape(data, dims): numpy.ndarray reshaped array """ - try: + if isinstance(data, (float, int)): + data = np.array(data) + elif isinstance(data, xr.DataArray): data = data.values - except AttributeError: - pass return data.reshape(dims) @@ -697,7 +692,7 @@ class ExtData(External, Data): """ def __init__(self, file_name, sheet, time_row_or_col, cell, - interp, coords, root, py_name): + interp, coords, root, final_coords, py_name): super().__init__(py_name) self.files = [file_name] self.sheets = [sheet] @@ -705,19 +700,17 @@ def __init__(self, file_name, sheet, time_row_or_col, cell, self.cells = [cell] self.coordss = [coords] self.root = root - self.interp = interp + self.final_coords = final_coords + self.interp = interp or "interpolate" self.is_float = not bool(coords) # check if the interpolation method is valid - if not interp: - self.interp = "interpolate" - if self.interp not in ["interpolate", "raw", - "look forward", "hold backward"]: + "look_forward", "hold_backward"]: raise ValueError(self.py_name + "\n" + " The interpolation method (interp) must be " + "'raw', 'interpolate', " - + "'look forward' or 'hold backward") + + "'look_forward' or 'hold_backward") def add(self, file_name, sheet, time_row_or_col, cell, interp, coords): @@ -730,9 +723,9 @@ def add(self, file_name, sheet, time_row_or_col, cell, self.cells.append(cell) self.coordss.append(coords) - if not interp: - interp = "interpolate" - if interp != self.interp: + interp = interp or "interpolate" + + if interp.replace(" ", "_") != self.interp: raise ValueError(self.py_name + "\n" + "Error matching interpolation method with " + "previously defined one") @@ -745,29 +738,56 @@ def initialize(self): """ Initialize all elements and create the self.data xarray.DataArray """ - self.data = utils.xrmerge(*[ - self._initialize_data("data") - for self.file, self.sheet, self.x_row_or_col, - self.cell, self.coords - in zip(self.files, self.sheets, self.time_row_or_cols, - self.cells, self.coordss)]) + if len(self.coordss) == 1: + # Just loag one value (no add) + for self.file, self.sheet, self.x_row_or_col,\ + self.cell, self.coords\ + in zip(self.files, self.sheets, self.time_row_or_cols, + self.cells, self.coordss): + self.data = self._initialize_data("data") + else: + # Load in several lines (add) + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + + for self.file, self.sheet, self.x_row_or_col,\ + self.cell, self.coords\ + in zip(self.files, self.sheets, self.time_row_or_cols, + self.cells, self.coordss): + values = self._initialize_data("data") + + coords = {"time": values.coords["time"].values, **self.coords} + if "time" not in self.data.dims: + self.data = self.data.expand_dims( + {"time": coords["time"]}, axis=0).copy() + + self.data.loc[coords] = values.values + + # set what to return when raw + if self.final_coords: + self.nan = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + else: + self.nan = np.nan -class ExtLookup(External): +class ExtLookup(External, Lookups): """ Class for Vensim GET XLS LOOKUPS/GET DIRECT LOOKUPS """ - def __init__(self, file_name, sheet, x_row_or_col, cell, - coords, root, py_name): + def __init__(self, file_name, sheet, x_row_or_col, cell, coords, + root, final_coords, py_name): super().__init__(py_name) self.files = [file_name] self.sheets = [sheet] self.x_row_or_cols = [x_row_or_col] self.cells = [cell] - self.root = root self.coordss = [coords] + self.root = root + self.final_coords = final_coords self.interp = "interpolate" + self.is_float = not bool(coords) def add(self, file_name, sheet, x_row_or_col, cell, coords): """ @@ -787,64 +807,33 @@ def initialize(self): """ Initialize all elements and create the self.data xarray.DataArray """ - self.data = utils.xrmerge(*[ - self._initialize_data("lookup") - for self.file, self.sheet, self.x_row_or_col, - self.cell, self.coords - in zip(self.files, self.sheets, self.x_row_or_cols, - self.cells, self.coordss)]) - - def __call__(self, x): - return self._call(self.data, x) - - def _call(self, data, x): - if isinstance(x, xr.DataArray): - if not x.dims: - # shape 0 xarrays - return self._call(data, float(x)) - if np.all(x > data['lookup_dim'].values[-1]): - outdata, _ = xr.broadcast(data[-1], x) - warnings.warn( - self.py_name + "\n" - + "extrapolating data above the maximum value of the series") - elif np.all(x < data['lookup_dim'].values[0]): - outdata, _ = xr.broadcast(data[0], x) - warnings.warn( - self.py_name + "\n" - + "extrapolating data below the minimum value of the series") - else: - data, _ = xr.broadcast(data, x) - outdata = data[0].copy() - for a in utils.xrsplit(x): - outdata.loc[a.coords] = self._call( - data.loc[a.coords], - float(a)) - # the output will be always an xarray - return outdata.reset_coords('lookup_dim', drop=True) - + if len(self.coordss) == 1: + # Just loag one value (no add) + for self.file, self.sheet, self.x_row_or_col,\ + self.cell, self.coords\ + in zip(self.files, self.sheets, self.x_row_or_cols, + self.cells, self.coordss): + self.data = self._initialize_data("lookup") else: - if x in data['lookup_dim'].values: - outdata = data.sel(lookup_dim=x) - elif x > data['lookup_dim'].values[-1]: - outdata = data[-1] - warnings.warn( - self.py_name + "\n" - + "extrapolating data above the maximum value of the series") - elif x < data['lookup_dim'].values[0]: - outdata = data[0] - warnings.warn( - self.py_name + "\n" - + "extrapolating data below the minimum value of the series") - else: - outdata = data.interp(lookup_dim=x) + # Load in several lines (add) + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) - # the output could be a float or an xarray - if self.coordss[0]: - # Remove lookup dimension coord from the DataArray - return outdata.reset_coords('lookup_dim', drop=True) - else: - # if lookup has no-coords return a float - return float(outdata) + for self.file, self.sheet, self.x_row_or_col,\ + self.cell, self.coords\ + in zip(self.files, self.sheets, self.x_row_or_cols, + self.cells, self.coordss): + values = self._initialize_data("lookup") + + coords = { + "lookup_dim": values.coords["lookup_dim"].values, + **self.coords + } + if "lookup_dim" not in self.data.dims: + self.data = self.data.expand_dims( + {"lookup_dim": coords["lookup_dim"]}, axis=0).copy() + + self.data.loc[coords] = values.values class ExtConstant(External): @@ -852,14 +841,17 @@ class ExtConstant(External): Class for Vensim GET XLS CONSTANTS/GET DIRECT CONSTANTS """ - def __init__(self, file_name, sheet, cell, coords, root, py_name): + def __init__(self, file_name, sheet, cell, coords, + root, final_coords, py_name): super().__init__(py_name) self.files = [file_name] self.sheets = [sheet] - self.transposes = [cell[-1] == '*'] + self.transposes = [ + cell[-1] == '*' and np.prod(utils.compute_shape(coords)) > 1] self.cells = [cell.strip('*')] - self.root = root self.coordss = [coords] + self.root = root + self.final_coords = final_coords def add(self, file_name, sheet, cell, coords): """ @@ -867,7 +859,8 @@ def add(self, file_name, sheet, cell, coords): """ self.files.append(file_name) self.sheets.append(sheet) - self.transposes.append(cell[-1] == '*') + self.transposes.append( + cell[-1] == '*' and np.prod(utils.compute_shape(coords)) > 1) self.cells.append(cell.strip('*')) self.coordss.append(coords) @@ -879,11 +872,22 @@ def initialize(self): """ Initialize all elements and create the self.data xarray.DataArray """ - self.data = utils.xrmerge(*[ - self._initialize() - for self.file, self.sheet, self.transpose, self.cell, self.coords - in zip(self.files, self.sheets, self.transposes, - self.cells, self.coordss)]) + if len(self.coordss) == 1: + # Just loag one value (no add) + for self.file, self.sheet, self.transpose, self.cell, self.coords\ + in zip(self.files, self.sheets, self.transposes, + self.cells, self.coordss): + self.data = self._initialize() + else: + # Load in several lines (add) + + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + + for self.file, self.sheet, self.transpose, self.cell, self.coords\ + in zip(self.files, self.sheets, self.transposes, + self.cells, self.coordss): + self.data.loc[self.coords] = self._initialize().values def _initialize(self): """ @@ -921,14 +925,14 @@ def _initialize(self): self.py_name + "\n" + "Constant value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(cell_type, self.cell) + + "\t{}:\t'{}'\n".format(cell_type, self.cell) ) elif self.missing == "raise": raise ValueError( self.py_name + "\n" + "Constant value missing or non-valid in:\n" + self._file_sheet - + "\t{}:\t{}\n".format(cell_type, self.cell) + + "\t{}:\t'{}'\n".format(cell_type, self.cell) ) # Create only an xarray if the data is not 0 dimensional diff --git a/pysd/py_backend/functions.py b/pysd/py_backend/functions.py index b609e48c..05349ffe 100644 --- a/pysd/py_backend/functions.py +++ b/pysd/py_backend/functions.py @@ -1,37 +1,38 @@ """ -These functions have no direct analog in the standard python data analytics -stack, or require information about the internal state of the system beyond -what is present in the function call. We provide them in a structure that -makes it easy for the model elements to call. +The provided functions have no direct analog in the standard Python data +analytics stack, or require information about the internal state of the +system beyond what is present in the function call. They are provided +in a structure that makes it easy for the model elements to call. The +functions may be similar to the original functions given by Vensim or +Stella, but sometimes the number or order of arguments may change. """ - import warnings import numpy as np import xarray as xr -import scipy.stats as stats small_vensim = 1e-6 # What is considered zero according to Vensim Help -def ramp(time, slope, start, finish=0): +def ramp(time, slope, start, finish=None): """ Implements vensim's and xmile's RAMP function. Parameters ---------- - time: function - The current time of modelling. + time: callable + Function that returns the current time. slope: float The slope of the ramp starting at zero at time start. start: float Time at which the ramp begins. - finish: float - Optional. Time at which the ramp ends. + finish: float or None (oprional) + Time at which the ramp ends. If None the ramp will never end. + Default is None. Returns ------- - response: float + float or xarray.DataArray: If prior to ramp start, returns zero. If after ramp ends, returns top of ramp. @@ -40,7 +41,7 @@ def ramp(time, slope, start, finish=0): if t < start: return 0 else: - if finish <= 0: + if finish is None: return slope * (t - start) elif t > finish: return slope * (finish - start) @@ -54,6 +55,8 @@ def step(time, value, tstep): Parameters ---------- + time: callable + Function that returns the current time. value: float The height of the step. tstep: float @@ -61,7 +64,7 @@ def step(time, value, tstep): Returns ------- - float: + float or xarray.DataArray: - In range [-inf, tstep): returns 0 - In range [tstep, +inf]: @@ -70,158 +73,52 @@ def step(time, value, tstep): return value if time() >= tstep else 0 -def pulse(time, start, duration): +def pulse(time, start, repeat_time=0, width=None, magnitude=None, end=None): """ - Implements vensim's PULSE function. + Implements Vensim's PULSE and PULSE TRAIN functions and Xmile's PULSE + function. Parameters ---------- - time: function + time: callable Function that returns the current time. start: float Starting time of the pulse. - duration: float - Duration of the pulse. - - Returns - ------- - float: - - In range [-inf, start): - returns 0 - - In range [start, start + duration): - returns 1 - - In range [start + duration, +inf]: - returns 0 - - """ - t = time() - return 1 if start <= t < start + duration else 0 - - -def pulse_train(time, start, duration, repeat_time, end): - """ - Implements vensim's PULSE TRAIN function. - - Parameters - ---------- - time: function - Function that returns the current time. - start: float - Starting time of the pulse. - duration: float - Duration of the pulse. - repeat_time: float - Time interval of the pulse repetition. - end: float - Final time of the pulse. + repeat_time: float (optional) + Time interval of the pulse repetition. If 0 it will return a + single pulse. Default is 0. + width: float or None (optional) + Duration of the pulse. If None only one-time_step pulse will be + generated. Default is None. + magnitude: float or None (optional) + The magnitude of the pulse. If None it will return 1 when the + pulse happens, similar to magnitude=time_step(). Default is None. + end: float or None (optional) + Final time of the pulse. If None there is no final time. + Default is None. Returns ------- - float: + float or xarray.DataArray: - In range [-inf, start): returns 0 - - In range [start + n*repeat_time, start + n*repeat_time + duration): - returns 1 - - In range [start + n*repeat_time + duration, - start + (n+1)*repeat_time): + - In range [start + n*repeat_time, start + n*repeat_time + width): + returns magnitude/time_step or 1 + - In range [start + n*repeat_time + width, start + (n+1)*repeat_time): returns 0 """ t = time() - if start <= t < end: - return 1 if (t - start) % repeat_time < duration else 0 + width = .5*time.time_step() if width is None else width + out = magnitude/time.time_step() if magnitude is not None else 1 + if repeat_time == 0: + return out if start - small_vensim <= t < start + width else 0 + elif start <= t and (end is None or t < end): + return out if (t - start + small_vensim) % repeat_time < width else 0 else: return 0 -def pulse_magnitude(time, magnitude, start, repeat_time=0): - """ - Implements xmile's PULSE function. Generate a one-DT wide pulse - at the given time. - - Parameters - ---------- - time: function - Function that returns the current time. - magnitude: - Magnitude of the pulse. - start: float - Starting time of the pulse. - repeat_time: float (optional) - Time interval of the pulse repetition. Default is 0, only one - pulse will be generated. - - Notes - ----- - PULSE(time(), 20, 12, 5) generates a pulse value of 20/DT at - time 12, 17, 22, etc. - - Returns - ------- - float: - - In rage [-inf, start): - returns 0 - - In range [start + n*repeat_time, start + n*repeat_time + dt): - returns magnitude/dt - - In rage [start + n*repeat_time + dt, start + (n+1)*repeat_time): - returns 0 - - """ - t = time() - if repeat_time <= small_vensim: - if abs(t - start) < time.time_step(): - return magnitude * time.time_step() - else: - return 0 - else: - if abs((t - start) % repeat_time) < time.time_step(): - return magnitude * time.time_step() - else: - return 0 - - -def lookup(x, xs, ys): - """ - Intermediate values are calculated with linear interpolation between - the intermediate points. Out-of-range values are the same as the - closest endpoint (i.e, no extrapolation is performed). - """ - return np.interp(x, xs, ys) - - -def lookup_extrapolation(x, xs, ys): - """ - Intermediate values are calculated with linear interpolation between - the intermediate points. Out-of-range values are calculated with linear - extrapolation from the last two values at either end. - """ - if x < xs[0]: - dx = xs[1] - xs[0] - dy = ys[1] - ys[0] - k = dy / dx - return ys[0] + (x - xs[0]) * k - if x > xs[-1]: - dx = xs[-1] - xs[-2] - dy = ys[-1] - ys[-2] - k = dy / dx - return ys[-1] + (x - xs[-1]) * k - return np.interp(x, xs, ys) - - -def lookup_discrete(x, xs, ys): - """ - Intermediate values take on the value associated with the next lower - x-coordinate (also called a step-wise function). The last two points - of a discrete graphical function must have the same y value. - Out-of-range values are the same as the closest endpoint - (i.e, no extrapolation is performed). - """ - for index in range(0, len(xs)): - if x < xs[index]: - return ys[index - 1] if index > 0 else ys[index] - return ys[-1] - - def if_then_else(condition, val_if_true, val_if_false): """ Implements Vensim's IF THEN ELSE function. @@ -230,17 +127,30 @@ def if_then_else(condition, val_if_true, val_if_false): Parameters ---------- condition: bool or xarray.DataArray of bools - val_if_true: function + val_if_true: callable Value to evaluate and return when condition is true. - val_if_false: function + val_if_false: callable Value to evaluate and return when condition is false. Returns ------- - The value depending on the condition. + float or xarray.DataArray: + The value depending on the condition. """ + # NUMPY: replace xr by np if isinstance(condition, xr.DataArray): + # NUMPY: neccessarry for keep the same shape always + # if condition.all(): + # value = val_if_true() + # elif not condition.any(): + # value = val_if_false() + # else: + # return np.where(condition, val_if_true(), val_if_false()) + # + # if isinstance(value, np.ndarray): + # return value + # return np.full_like(condition, value) if condition.all(): return val_if_true() elif not condition.any(): @@ -251,49 +161,7 @@ def if_then_else(condition, val_if_true, val_if_false): return val_if_true() if condition else val_if_false() -def logical_and(*args): - """ - Implements Vensim's :AND: method for two or several arguments. - - Parameters - ---------- - *args: arguments - The values to compare with and operator - - Returns - ------- - result: bool or xarray.DataArray - The result of the comparison. - - """ - current = args[0] - for arg in args[1:]: - current = np.logical_and(arg, current) - return current - - -def logical_or(*args): - """ - Implements Vensim's :OR: method for two or several arguments. - - Parameters - ---------- - *args: arguments - The values to compare with and operator - - Returns - ------- - result: bool or xarray.DataArray - The result of the comparison. - - """ - current = args[0] - for arg in args[1:]: - current = np.logical_or(arg, current) - return current - - -def xidz(numerator, denominator, value_if_denom_is_zero): +def xidz(numerator, denominator, x): """ Implements Vensim's XIDZ function. https://www.vensim.com/documentation/fn_xidz.htm @@ -304,26 +172,35 @@ def xidz(numerator, denominator, value_if_denom_is_zero): Parameters ---------- numerator: float or xarray.DataArray + Numerator of the operation. denominator: float or xarray.DataArray - Components of the division operation - value_if_denom_is_zero: float or xarray.DataArray - The value to return if the denominator is zero + Denominator of the operation. + x: float or xarray.DataArray + The value to return if the denominator is zero. Returns ------- - numerator / denominator if denominator > 1e-6 - otherwise, returns value_if_denom_is_zero + float or xarray.DataArray: + - numerator/denominator if denominator > small_vensim + - value_if_denom_is_zero otherwise """ + # NUMPY: replace DataArray by np.ndarray, xr.where -> np.where if isinstance(denominator, xr.DataArray): return xr.where(np.abs(denominator) < small_vensim, - value_if_denom_is_zero, - numerator * 1.0 / denominator) + x, + numerator/denominator) if abs(denominator) < small_vensim: - return value_if_denom_is_zero + # NUMPY: neccessarry for keep the same shape always + # if isinstance(numerator, np.ndarray): + # return np.full_like(numerator, x) + return x else: - return numerator * 1.0 / denominator + # NUMPY: neccessarry for keep the same shape always + # if isinstance(x, np.ndarray): + # return np.full_like(x, numerator/denominator) + return numerator/denominator def zidz(numerator, denominator): @@ -341,50 +218,54 @@ def zidz(numerator, denominator): Returns ------- - result of division numerator/denominator if denominator is not zero, - otherwise zero. + float or xarray.DataArray: + - numerator/denominator if denominator > small_vensim + - 0 or 0s array otherwise """ + # NUMPY: replace DataArray by np.ndarray, xr.where -> np.where if isinstance(denominator, xr.DataArray): return xr.where(np.abs(denominator) < small_vensim, 0, - numerator * 1.0 / denominator) + numerator/denominator) if abs(denominator) < small_vensim: + # NUMPY: neccessarry for keep the same shape always + # if isinstance(denominator, np.ndarray): + # return np.zeros_like(denominator) + if isinstance(numerator, xr.DataArray): + return xr.DataArray(0, numerator.coords, numerator.dims) return 0 else: - return numerator * 1.0 / denominator + return numerator/denominator -def active_initial(time, expr, init_val): +def active_initial(stage, expr, init_val): """ Implements vensim's ACTIVE INITIAL function + Parameters ---------- - time: function - The current time function - expr - init_val + stage: str + The stage of the model. + expr: callable + Running stage value + init_val: float or xarray.DataArray + Initialization stage value. Returns ------- - + float or xarray.DataArray: + - inti_val if stage='Initialization' + - expr() otherwise """ - if time.stage == 'Initialization': + # NUMPY: both must have same dimensions in inputs, remove time.stage + if stage == 'Initialization': return init_val else: return expr() -def bounded_normal(minimum, maximum, mean, std, seed): - """ - Implements vensim's BOUNDED NORMAL function - """ - # np.random.seed(seed) - # we could bring this back later, but for now, ignore - return stats.truncnorm.rvs(minimum, maximum, loc=mean, scale=std) - - def incomplete(*args): warnings.warn( 'Call to undefined function, calling dependencies and returning NaN', @@ -394,27 +275,7 @@ def incomplete(*args): def not_implemented_function(*args): - raise NotImplementedError( - 'Not implemented function {}'.format(args[0])) - - -def log(x, base): - """ - Implements Vensim's LOG function with change of base. - - Parameters - ---------- - x: float or xarray.DataArray - Input value. - base: float or xarray.DataArray - Base of the logarithm. - - Returns - ------- - float - The log of 'x' in base 'base'. - """ - return np.log(x) / np.log(base) + raise NotImplementedError(f"Not implemented function '{args[0]}'") def integer(x): @@ -428,9 +289,11 @@ def integer(x): Returns ------- - Returns integer part of x. + integer: float or xarray.DataArray + Returns integer part of x. """ + # NUMPY: replace xr by np if isinstance(x, xr.DataArray): return x.astype(int) else: @@ -454,6 +317,7 @@ def quantum(a, b): If b > 0 returns b * integer(a/b). Otherwise, returns a. """ + # NUMPY: replace xr by np if isinstance(b, xr.DataArray): return xr.where(b < small_vensim, a, b*integer(a/b)) if b < small_vensim: @@ -475,8 +339,9 @@ def modulo(x, m): Returns ------- - Returns x modulo m, if x is smaller than 0 the result is given in - the range (-m, 0] as Vensim does. x - quantum(x, m) + modulo: float or xarray.DataArray + Returns x modulo m, if x is smaller than 0 the result is given + in the range (-m, 0] as Vensim does. x - quantum(x, m) """ return x - quantum(x, m) @@ -489,17 +354,18 @@ def sum(x, dim=None): Parameters ---------- x: xarray.DataArray - Input value. + Input value. dim: list of strs (optional) - Dimensions to apply the function over. - If not given the function will be applied over all dimensions. + Dimensions to apply the function over. + If not given the function will be applied over all dimensions. Returns ------- - xarray.DataArray or float - The result of the sum operation in the given dimensions. + sum: xarray.DataArray or float + The result of the sum operation in the given dimensions. """ + # NUMPY: replace by np.sum(x, axis=axis) put directly in the file # float returned if the function is applied over all the dimensions if dim is None or set(x.dims) == set(dim): return float(x.sum()) @@ -514,17 +380,18 @@ def prod(x, dim=None): Parameters ---------- x: xarray.DataArray - Input value. + Input value. dim: list of strs (optional) - Dimensions to apply the function over. - If not given the function will be applied over all dimensions. + Dimensions to apply the function over. + If not given the function will be applied over all dimensions. Returns ------- - xarray.DataArray or float - The result of the product operation in the given dimensions. + prod: xarray.DataArray or float + The result of the product operation in the given dimensions. """ + # NUMPY: replace by np.prod(x, axis=axis) put directly in the file # float returned if the function is applied over all the dimensions if dim is None or set(x.dims) == set(dim): return float(x.prod()) @@ -539,17 +406,18 @@ def vmin(x, dim=None): Parameters ---------- x: xarray.DataArray - Input value. + Input value. dim: list of strs (optional) - Dimensions to apply the function over. - If not given the function will be applied over all dimensions. + Dimensions to apply the function over. + If not given the function will be applied over all dimensions. Returns ------- - xarray.DataArray or float - The result of the minimum value over the given dimensions. + vmin: xarray.DataArray or float + The result of the minimum value over the given dimensions. """ + # NUMPY: replace by np.min(x, axis=axis) put directly in the file # float returned if the function is applied over all the dimensions if dim is None or set(x.dims) == set(dim): return float(x.min()) @@ -564,17 +432,18 @@ def vmax(x, dim=None): Parameters ---------- x: xarray.DataArray - Input value. + Input value. dim: list of strs (optional) - Dimensions to apply the function over. - If not given the function will be applied over all dimensions. + Dimensions to apply the function over. + If not given the function will be applied over all dimensions. Returns ------- - xarray.DataArray or float - The result of the maximum value over the dimensions. + vmax: xarray.DataArray or float + The result of the maximum value over the dimensions. """ + # NUMPY: replace by np.max(x, axis=axis) put directly in the file # float returned if the function is applied over all the dimensions if dim is None or set(x.dims) == set(dim): return float(x.max()) @@ -599,4 +468,6 @@ def invert_matrix(mat): Inverted matrix. """ + # NUMPY: avoid converting to xarray, put directly the expression + # in the model return xr.DataArray(np.linalg.inv(mat.values), mat.coords, mat.dims) diff --git a/pysd/py_backend/lookups.py b/pysd/py_backend/lookups.py new file mode 100644 index 00000000..4f86571a --- /dev/null +++ b/pysd/py_backend/lookups.py @@ -0,0 +1,203 @@ +import warnings + +import pandas as pd +import numpy as np +import xarray as xr + +from . import utils + + +class Lookups(object): + def set_values(self, values): + """Set new values from user input""" + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + + if isinstance(values, pd.Series): + index = list(values.index) + index.sort() + self.data = self.data.expand_dims( + {'lookup_dim': index}, axis=0).copy() + + for index, value in values.items(): + if isinstance(values.values[0], xr.DataArray): + self.data.loc[index].loc[value.coords] =\ + value + else: + self.data.loc[index] = value + else: + if isinstance(values, xr.DataArray): + self.data.loc[values.coords] = values.values + else: + if self.final_coords: + self.data.loc[:] = values + else: + self.data = values + + def __call__(self, x, final_subs=None): + try: + return self._call(self.data, x, final_subs) + except (TypeError, KeyError): + # this except catch the errors when a lookups has been + # changed to a constant value by the user + if final_subs and isinstance(self.data, xr.DataArray): + # self.data is an array, reshape it + outdata = xr.DataArray(np.nan, final_subs, list(final_subs)) + return xr.broadcast(outdata, self.data)[1] + elif final_subs: + # self.data is a float, create an array + return xr.DataArray(self.data, final_subs, list(final_subs)) + else: + return self.data + + def _call(self, data, x, final_subs=None): + if isinstance(x, xr.DataArray): + if not x.dims: + # shape 0 xarrays + return self._call(data, float(x)) + + outdata = xr.DataArray(np.nan, final_subs, list(final_subs)) + + if self.interp != "extrapolate" and\ + np.all(x > data['lookup_dim'].values[-1]): + outdata_ext = data[-1] + warnings.warn( + self.py_name + "\n" + + "extrapolating data above the maximum value of the series") + elif self.interp != "extrapolate" and\ + np.all(x < data['lookup_dim'].values[0]): + outdata_ext = data[0] + warnings.warn( + self.py_name + "\n" + + "extrapolating data below the minimum value of the series") + else: + data = xr.broadcast(data, x)[0] + for a in utils.xrsplit(x): + outdata.loc[a.coords] = self._call( + data.loc[a.coords], + float(a)) + return outdata + + # return the final array in the specified dimensions order + return xr.broadcast( + outdata, outdata_ext.reset_coords('lookup_dim', drop=True))[1] + + else: + if x in data['lookup_dim'].values: + outdata = data.sel(lookup_dim=x) + elif x > data['lookup_dim'].values[-1]: + if self.interp == "extrapolate": + # extrapolate method for xmile models + k = (data[-1]-data[-2])\ + / (data['lookup_dim'].values[-1] + - data['lookup_dim'].values[-2]) + outdata = data[-1] + k*(x - data['lookup_dim'].values[-1]) + else: + outdata = data[-1] + warnings.warn( + self.py_name + "\n" + + "extrapolating data above the maximum value of the series") + elif x < data['lookup_dim'].values[0]: + if self.interp == "extrapolate": + # extrapolate method for xmile models + k = (data[1]-data[0])\ + / (data['lookup_dim'].values[1] + - data['lookup_dim'].values[0]) + outdata = data[0] + k*(x - data['lookup_dim'].values[0]) + else: + outdata = data[0] + warnings.warn( + self.py_name + "\n" + + "extrapolating data below the minimum value of the series") + elif self.interp == 'hold_backward': + outdata = data.sel(lookup_dim=x, method="pad") + else: + outdata = data.interp(lookup_dim=x) + + # the output could be a float or an xarray + if self.is_float: + # if lookup has no-coords return a float + return float(outdata) + else: + # Remove lookup dimension coord from the DataArray + return outdata.reset_coords('lookup_dim', drop=True) + + +class HardcodedLookups(Lookups): + """Class for lookups defined in the file""" + + def __init__(self, x, y, coords, interp, final_coords, py_name): + # TODO: avoid add and merge all declarations in one definition + self.is_float = not bool(coords) + self.py_name = py_name + self.final_coords = final_coords + self.values = [(x, y, coords)] + self.interp = interp + + def add(self, x, y, coords): + self.values.append((x, y, coords)) + + def initialize(self): + """ + Initialize all elements and create the self.data xarray.DataArray + """ + if len(self.values) == 1: + # Just loag one value (no add) + for x, y, coords in self.values: + y = np.array(y).reshape((len(x),) + (1,)*len(coords)) + self.data = xr.DataArray( + np.tile(y, [1] + utils.compute_shape(coords)), + {"lookup_dim": x, **coords}, + ["lookup_dim"] + list(coords) + ) + else: + # Load in several lines (add) + self.data = xr.DataArray( + np.nan, self.final_coords, list(self.final_coords)) + + for x, y, coords in self.values: + if "lookup_dim" not in self.data.dims: + # include lookup_dim dimension in the final array + self.data = self.data.expand_dims( + {"lookup_dim": x}, axis=0).copy() + else: + # add new coordinates (if needed) to lookup_dim + x_old = list(self.data.lookup_dim.values) + x_new = list(set(x).difference(x_old)) + self.data = self.data.reindex(lookup_dim=x_old+x_new) + + # reshape y value and assign it to self.data + y = np.array(y).reshape((len(x),) + (1,)*len(coords)) + self.data.loc[{"lookup_dim": x, **coords}] =\ + np.tile(y, [1] + utils.compute_shape(coords)) + + # sort data + self.data = self.data.sortby("lookup_dim") + + if np.any(np.isnan(self.data)): + # fill missing values of different input lookup_dim values + values = self.data.values + self._fill_missing(self.data.lookup_dim.values, values) + self.data = xr.DataArray(values, self.data.coords, self.data.dims) + + def _fill_missing(self, series, data): + """ + Fills missing values in lookups to have a common series. + Mutates the values in data. + + Returns + ------- + None + + """ + if len(data.shape) > 1: + # break the data array until arrive to a vector + for i in range(data.shape[1]): + if np.any(np.isnan(data[:, i])): + self._fill_missing(series, data[:, i]) + elif not np.all(np.isnan(data)): + # interpolate missing values + data[np.isnan(data)] = np.interp( + series[np.isnan(data)], + series[~np.isnan(data)], + data[~np.isnan(data)]) diff --git a/pysd/py_backend/model.py b/pysd/py_backend/model.py new file mode 100644 index 00000000..59aa3194 --- /dev/null +++ b/pysd/py_backend/model.py @@ -0,0 +1,1709 @@ +""" +Macro and Model classes are the main classes for loading and interacting +with a PySD model. Model class allows loading and running a PySD model. +Several methods and propierties are inherited from Macro class, which +allows integrating a model or a Macro expression (set of functions in +a separate file). +""" +import warnings +import inspect +import pickle +from typing import Union + +import numpy as np +import xarray as xr +import pandas as pd + +from . import utils +from .statefuls import DynamicStateful, Stateful +from .external import External, Excels +from .cache import Cache, constant_cache +from .data import TabData +from .lookups import HardcodedLookups +from .components import Components, Time + +from pysd._version import __version__ + + +class Macro(DynamicStateful): + """ + The Macro class implements a stateful representation of the system, + and contains the majority of methods for accessing and modifying + components. + + When the instance in question also serves as the root model object + (as opposed to a macro or submodel within another model) it will have + added methods to facilitate execution. + + The Macro object will be created with components drawn from a + translated Python model file. + + Parameters + ---------- + py_model_file: str or pathlib.Path + Filename of a model or macro which has already been converted + into a Python format. + params: dict or None (optional) + Dictionary of the macro parameters. Default is None. + return_func: str or None (optional) + The name of the function to return from the macro. Default is None. + time: components.Time or None (optional) + Time object for integration. If None a new time object will + be generated (for models), if passed the time object will be + used (for macros). Default is None. + time_initialization: callable or None + Time to set at the begginning of the Macro. Default is None. + data_files: dict or list or str or None + The dictionary with keys the name of file and variables to + load the data from there. Or the list of names or name of the + file to search the data in. Only works for TabData type object + and it is neccessary to provide it. Default is None. + py_name: str or None + The name of the Macro object. Default is None. + + """ + def __init__(self, py_model_file, params=None, return_func=None, + time=None, time_initialization=None, data_files=None, + py_name=None): + super().__init__() + self.time = time + self.time_initialization = time_initialization + self.cache = Cache() + self.py_name = py_name + self.external_loaded = False + self.lookups_loaded = False + self.components = Components(str(py_model_file), self.set_components) + + if __version__.split(".")[0]\ + != self.get_pysd_compiler_version().split(".")[0]: + raise ImportError( + "\n\nNot able to import the model. " + + "The model was translated with a " + + "not compatible version of PySD:" + + "\n\tPySD " + self.get_pysd_compiler_version() + + "\n\nThe current version of PySd is:" + + "\n\tPySD " + __version__ + "\n\n" + + "Please translate again the model with the function" + + " read_vensim or read_xmile.") + + self._namespace = self.components._components.component.namespace + self._dependencies =\ + self.components._components.component.dependencies.copy() + self._subscript_dict = getattr( + self.components._components, "_subscript_dict", {}) + self._modules = getattr( + self.components._components, "_modules", {}) + + self._doc = self._build_doc() + + if params is not None: + # add params to namespace + self._namespace.update(self.components._components._params) + # create new components with the params + self.set_components(params, new=True) + # update dependencies + for param in params: + self._dependencies[ + self._namespace[param]] = {"time"} + + # Get the collections of stateful elements and external elements + self._stateful_elements = { + name: getattr(self.components, name) + for name in dir(self.components) + if isinstance(getattr(self.components, name), Stateful) + } + self._dynamicstateful_elements = [ + getattr(self.components, name) for name in dir(self.components) + if isinstance(getattr(self.components, name), DynamicStateful) + ] + self._external_elements = [ + getattr(self.components, name) for name in dir(self.components) + if isinstance(getattr(self.components, name), External) + ] + self._macro_elements = [ + getattr(self.components, name) for name in dir(self.components) + if isinstance(getattr(self.components, name), Macro) + ] + + self._data_elements = [ + getattr(self.components, name) for name in dir(self.components) + if isinstance(getattr(self.components, name), TabData) + ] + + self._lookup_elements = [ + getattr(self.components, name) for name in dir(self.components) + if isinstance(getattr(self.components, name), HardcodedLookups) + ] + + if data_files: + self._get_data(data_files) + + self._assign_cache_type() + self._get_initialize_order() + + if return_func is not None: + self.return_func = getattr(self.components, return_func) + else: + self.return_func = lambda: 0 + + self.py_model_file = str(py_model_file) + + def __call__(self): + return self.return_func() + + @property + def doc(self) -> pd.DataFrame: + """ + The documentation of the model. + """ + return self._doc.copy() + + @property + def namespace(self) -> dict: + """ + The namespace dictionary of the model. + """ + return self._namespace.copy() + + @property + def dependencies(self) -> dict: + """ + The dependencies dictionary of the model. + """ + return self._dependencies.copy() + + @property + def subscripts(self) -> dict: + """ + The subscripts dictionary of the model. + """ + return self._subscript_dict.copy() + + @property + def modules(self) -> Union[dict, None]: + """ + The dictionary of modules of the model. If the model is not + split by modules it returns None. + """ + return self._modules.copy() or None + + def clean_caches(self): + """ + Clean the cahce of the object and the macros objects that it + contains + """ + self.cache.clean() + # if nested macros + [macro.clean_caches() for macro in self._macro_elements] + + def _get_data(self, data_files): + if isinstance(data_files, dict): + for data_file, vars in data_files.items(): + for var in vars: + found = False + for element in self._data_elements: + if var in [element.py_name, element.real_name]: + element.load_data(data_file) + found = True + break + if not found: + raise ValueError( + f"'{var}' not found as model data variable") + + else: + for element in self._data_elements: + element.load_data(data_files) + + def _get_initialize_order(self): + """ + Get the initialization order of the stateful elements + and their the full dependencies. + """ + # get the full set of dependencies to initialize an stateful object + # includying all levels + self.stateful_initial_dependencies = { + ext: set() + for ext in self._dependencies + if (ext.startswith("_") and not ext.startswith("_active_initial_")) + } + for element in self.stateful_initial_dependencies: + self._get_full_dependencies( + element, self.stateful_initial_dependencies[element], + "initial") + + # get the full dependencies of stateful objects taking into account + # only other objects + current_deps = { + element: [ + dep for dep in deps + if dep in self.stateful_initial_dependencies + ] for element, deps in self.stateful_initial_dependencies.items() + } + + # get initialization order of the stateful elements + self.initialize_order = [] + delete = True + while delete: + delete = [] + for element in current_deps: + if not current_deps[element]: + # if stateful element has no deps on others + # add to the queue to initialize + self.initialize_order.append(element) + delete.append(element) + for element2 in current_deps: + # remove dependency on the initialized element + if element in current_deps[element2]: + current_deps[element2].remove(element) + # delete visited elements + for element in delete: + del current_deps[element] + + if current_deps: + # if current_deps is not an empty set there is a circular + # reference between stateful objects + raise ValueError( + 'Circular initialization...\n' + + 'Not able to initialize the following objects:\n\t' + + '\n\t'.join(current_deps)) + + def _get_full_dependencies(self, element, dep_set, stateful_deps): + """ + Get all dependencies of an element, i.e., also get the dependencies + of the dependencies. When finding an stateful element only dependencies + for initialization are considered. + + Parameters + ---------- + element: str + Element to get the full dependencies. + dep_set: set + Set to include the dependencies of the element. + stateful_deps: "initial" or "step" + The type of dependencies to take in the case of stateful objects. + + Returns + ------- + None + + """ + deps = self._dependencies[element] + if element.startswith("_"): + deps = deps[stateful_deps] + for dep in deps: + if dep not in dep_set and not dep.startswith("__")\ + and dep != "time": + dep_set.add(dep) + self._get_full_dependencies(dep, dep_set, stateful_deps) + + def _add_constant_cache(self): + self.constant_funcs = set() + for element, cache_type in self.cache_type.items(): + if cache_type == "run": + self.components._set_component( + element, + constant_cache(getattr(self.components, element)) + ) + self.constant_funcs.add(element) + + def _remove_constant_cache(self): + for element in self.constant_funcs: + self.components._set_component( + element, + getattr(self.components, element).function) + self.constant_funcs = set() + + def _assign_cache_type(self): + """ + Assigns the cache type to all the elements from the namespace. + """ + self.cache_type = {"time": None} + + for element in self._namespace.values(): + if element not in self.cache_type\ + and element in self._dependencies: + self._assign_cache(element) + + for element, cache_type in self.cache_type.items(): + if cache_type is not None: + if element not in self.cache.cached_funcs\ + and self._count_calls(element) > 1: + self.components._set_component( + element, + self.cache(getattr(self.components, element))) + self.cache.cached_funcs.add(element) + + def _count_calls(self, element): + n_calls = 0 + for subelement in self._dependencies: + if subelement.startswith("_") and\ + element in self._dependencies[subelement]["step"]: + if element in\ + self._dependencies[subelement]["initial"]: + n_calls +=\ + 2*self._dependencies[subelement]["step"][element] + else: + n_calls +=\ + self._dependencies[subelement]["step"][element] + elif (not subelement.startswith("_") and + element in self._dependencies[subelement]): + n_calls +=\ + self._dependencies[subelement][element] + + return n_calls + + def _assign_cache(self, element): + """ + Assigns the cache type to the given element and its dependencies if + needed. + + Parameters + ---------- + element: str + Element name. + + Returns + ------- + None + + """ + if not self._dependencies[element]: + self.cache_type[element] = "run" + elif "__lookup__" in self._dependencies[element]: + self.cache_type[element] = None + elif self._isdynamic(self._dependencies[element]): + self.cache_type[element] = "step" + else: + self.cache_type[element] = "run" + for subelement in self._dependencies[element]: + if subelement.startswith("_initial_")\ + or subelement.startswith("__"): + continue + if subelement not in self.cache_type: + self._assign_cache(subelement) + if self.cache_type[subelement] == "step": + self.cache_type[element] = "step" + break + + def _isdynamic(self, dependencies): + """ + + Parameters + ---------- + dependencies: iterable + List of dependencies. + + Returns + ------- + isdynamic: bool + True if 'time' or a dynamic stateful objects is in dependencies. + + """ + if "time" in dependencies: + return True + for dep in dependencies: + if dep.startswith("_") and not dep.startswith("_initial_")\ + and not dep.startswith("__"): + return True + return False + + def get_pysd_compiler_version(self): + """ + Returns the version of pysd complier that used for generating + this model + """ + return self.components.__pysd_version__ + + def initialize(self): + """ + This function initializes the external objects and stateful objects + in the given order. + """ + # Initialize time + if self.time is None: + self.time = self.time_initialization() + + # Reset time to the initial one + self.time.reset() + self.cache.clean() + + self.components._init_outer_references({ + 'scope': self, + 'time': self.time + }) + + if not self.lookups_loaded: + # Initialize HardcodedLookups elements + for element in self._lookup_elements: + element.initialize() + + self.lookups_loaded = True + + if not self.external_loaded: + # Initialize external elements + for element in self._external_elements: + element.initialize() + + # Remove Excel data from memory + Excels.clean() + + self.external_loaded = True + + # Initialize stateful objects + for element_name in self.initialize_order: + self._stateful_elements[element_name].initialize() + + def ddt(self): + return np.array([component.ddt() for component + in self._dynamicstateful_elements], dtype=object) + + @property + def state(self): + return np.array([component.state for component + in self._dynamicstateful_elements], dtype=object) + + @state.setter + def state(self, new_value): + [component.update(val) for component, val + in zip(self._dynamicstateful_elements, new_value)] + + def export(self, file_name): + """ + Export stateful values to pickle file. + + Parameters + ---------- + file_name: str + Name of the file to export the values. + + """ + warnings.warn( + "\nCompatibility of exported states could be broken between" + " different versions of PySD or xarray, current versions:\n" + f"\tPySD {__version__}\n\txarray {xr.__version__}\n" + ) + stateful_elements = { + name: element.export() + for name, element in self._stateful_elements.items() + } + + with open(file_name, 'wb') as file: + pickle.dump( + (self.time(), + stateful_elements, + {'pysd': __version__, 'xarray': xr.__version__} + ), file) + + def import_pickle(self, file_name): + """ + Import stateful values from pickle file. + + Parameters + ---------- + file_name: str + Name of the file to import the values from. + + """ + with open(file_name, 'rb') as file: + time, stateful_dict, metadata = pickle.load(file) + + if __version__ != metadata['pysd']\ + or xr.__version__ != metadata['xarray']: # pragma: no cover + warnings.warn( + "\nCompatibility of exported states could be broken between" + " different versions of PySD or xarray. Current versions:\n" + f"\tPySD {__version__}\n\txarray {xr.__version__}\n" + "Loaded versions:\n" + f"\tPySD {metadata['pysd']}\n\txarray {metadata['xarray']}\n" + ) + + self.set_stateful(stateful_dict) + self.time.set_control_vars(initial_time=time) + + def get_args(self, param): + """ + Returns the arguments of a model element. + + Parameters + ---------- + param: str or func + The model element name or function. + + Returns + ------- + args: list + List of arguments of the function. + + Examples + -------- + >>> model.get_args('birth_rate') + >>> model.get_args('Birth Rate') + + """ + if isinstance(param, str): + func_name = utils.get_key_and_value_by_insensitive_key_or_value( + param, + self._namespace)[1] or param + + func = getattr(self.components, func_name) + else: + func = param + + if hasattr(func, 'args'): + # cached functions + return func.args + else: + # regular functions + args = inspect.getfullargspec(func)[0] + if 'self' in args: + args.remove('self') + return args + + def get_coords(self, param): + """ + Returns the coordinates and dims of a model element. + + Parameters + ---------- + param: str or func + The model element name or function. + + Returns + ------- + (coords, dims) or None: (dict, list) or None + The coords and the dimensions of the element if it has. + Otherwise, returns None. + + Examples + -------- + >>> model.get_coords('birth_rate') + >>> model.get_coords('Birth Rate') + + """ + if isinstance(param, str): + func_name = utils.get_key_and_value_by_insensitive_key_or_value( + param, + self._namespace)[1] or param + + func = getattr(self.components, func_name) + + else: + func = param + + if hasattr(func, "subscripts"): + dims = func.subscripts + if not dims: + return None + coords = {dim: self.components._subscript_dict[dim] + for dim in dims} + return coords, dims + elif hasattr(func, "state") and isinstance(func.state, xr.DataArray): + value = func() + else: + return None + + dims = list(value.dims) + coords = {coord: list(value.coords[coord].values) + for coord in value.coords} + return coords, dims + + def __getitem__(self, param): + """ + Returns the current value of a model component. + + Parameters + ---------- + param: str or func + The model element name. + + Returns + ------- + value: float or xarray.DataArray + The value of the model component. + + Examples + -------- + >>> model['birth_rate'] + >>> model['Birth Rate'] + + Note + ---- + It will crash if the model component takes arguments. + + """ + func_name = utils.get_key_and_value_by_insensitive_key_or_value( + param, + self._namespace)[1] or param + + if self.get_args(getattr(self.components, func_name)): + raise ValueError( + "Trying to get the current value of a lookup " + "to get all the values with the series data use " + "model.get_series_data(param)\n\n") + + return getattr(self.components, func_name)() + + def get_series_data(self, param): + """ + Returns the original values of a model lookup/data component. + + Parameters + ---------- + param: str + The model lookup/data element name. + + Returns + ------- + value: xarray.DataArray + Array with the value of the interpolating series + in the first dimension. + + Examples + -------- + >>> model['room_temperature'] + >>> model['Room temperature'] + + """ + func_name = utils.get_key_and_value_by_insensitive_key_or_value( + param, + self._namespace)[1] or param + + if func_name.startswith("_ext_"): + return getattr(self.components, func_name).data + elif "__data__" in self._dependencies[func_name]: + return getattr( + self.components, + self._dependencies[func_name]["__data__"] + ).data + elif "__lookup__" in self._dependencies[func_name]: + return getattr( + self.components, + self._dependencies[func_name]["__lookup__"] + ).data + else: + raise ValueError( + "Trying to get the values of a constant variable. " + "'model.get_series_data' only works lookups/data objects.\n\n") + + def set_components(self, params, new=False): + """ Set the value of exogenous model elements. + Element values can be passed as keyword=value pairs in the + function call. Values can be numeric type or pandas Series. + Series will be interpolated by integrator. + + Examples + -------- + >>> model.set_components({'birth_rate': 10}) + >>> model.set_components({'Birth Rate': 10}) + + >>> br = pandas.Series(index=range(30), values=np.sin(range(30)) + >>> model.set_components({'birth_rate': br}) + + + """ + # TODO: allow the params argument to take a pandas dataframe, where + # column names are variable names. However some variables may be + # constant or have no values for some index. This should be processed. + # TODO: make this compatible with loading outputs from other files + + for key, value in params.items(): + func_name = utils.get_key_and_value_by_insensitive_key_or_value( + key, + self._namespace)[1] + + if isinstance(value, np.ndarray) or isinstance(value, list): + raise TypeError( + 'When setting ' + key + '\n' + 'Setting subscripted must be done using a xarray.DataArray' + ' with the correct dimensions or a constant value ' + '(https://pysd.readthedocs.io/en/master/' + 'getting_started.html)') + + if func_name is None: + raise NameError( + "\n'%s' is not recognized as a model component." + % key) + + if new: + func = None + dims = None + else: + func = getattr(self.components, func_name) + _, dims = self.get_coords(func) or (None, None) + + # if the variable is a lookup or a data we perform the change in + # the object they call + func_type = getattr(func, "type", None) + if func_type in ["Lookup", "Data"]: + # getting the object from original dependencies + obj = self._dependencies[func_name][f"__{func_type.lower()}__"] + getattr( + self.components, + obj + ).set_values(value) + + # Update dependencies + if func_type == "Data": + if isinstance(value, pd.Series): + self._dependencies[func_name] = { + "time": 1, "__data__": obj + } + else: + self._dependencies[func_name] = {"__data__": obj} + + continue + + if isinstance(value, pd.Series): + new_function, deps = self._timeseries_component( + value, dims) + self._dependencies[func_name] = deps + elif callable(value): + new_function = value + # Using step cache adding time as dependency + # TODO it would be better if we can parse the content + # of the function to get all the dependencies + self._dependencies[func_name] = {"time": 1} + + else: + new_function = self._constant_component(value, dims) + self._dependencies[func_name] = {} + + # this won't handle other statefuls... + if '_integ_' + func_name in dir(self.components): + warnings.warn("Replacing the equation of stock" + + "{} with params".format(key), + stacklevel=2) + + new_function.__name__ = func_name + if dims: + new_function.dims = dims + self.components._set_component(func_name, new_function) + if func_name in self.cache.cached_funcs: + self.cache.cached_funcs.remove(func_name) + + def _timeseries_component(self, series, dims): + """ Internal function for creating a timeseries model element """ + # this is only called if the set_component function recognizes a + # pandas series + # TODO: raise a warning if extrapolating from the end of the series. + # TODO: data type variables should be creted using a Data object + # lookup type variables should be created using a Lookup object + + if isinstance(series.values[0], xr.DataArray): + # the interpolation will be time dependent + return lambda: utils.rearrange(xr.concat( + series.values, + series.index).interp(concat_dim=self.time()).reset_coords( + 'concat_dim', drop=True), + dims, self._subscript_dict), {'time': 1} + + elif dims: + # the interpolation will be time dependent + return lambda: utils.rearrange( + np.interp(self.time(), series.index, series.values), + dims, self._subscript_dict), {'time': 1} + + else: + # the interpolation will be time dependent + return lambda:\ + np.interp(self.time(), series.index, series.values),\ + {'time': 1} + + def _constant_component(self, value, dims): + """ Internal function for creating a constant model element """ + if dims: + return lambda: utils.rearrange( + value, dims, self._subscript_dict) + + else: + return lambda: value + + def set_initial_value(self, t, initial_value): + """ Set the system initial value. + + Parameters + ---------- + t : numeric + The system time + + initial_value : dict + A (possibly partial) dictionary of the system initial values. + The keys to this dictionary may be either pysafe names or + original model file names + + """ + self.time.set_control_vars(initial_time=t) + stateful_name = "_NONE" + modified_statefuls = set() + + for key, value in initial_value.items(): + component_name =\ + utils.get_key_and_value_by_insensitive_key_or_value( + key, self._namespace)[1] + if component_name is not None: + if self._dependencies[component_name]: + deps = list(self._dependencies[component_name]) + if len(deps) == 1 and deps[0] in self.initialize_order: + stateful_name = deps[0] + else: + component_name = key + stateful_name = key + + try: + _, dims = self.get_coords(component_name) + except TypeError: + dims = None + + if isinstance(value, xr.DataArray)\ + and not set(value.dims).issubset(set(dims)): + raise ValueError( + f"\nInvalid dimensions for {component_name}." + f"It should be a subset of {dims}, " + f"but passed value has {list(value.dims)}") + + if isinstance(value, np.ndarray) or isinstance(value, list): + raise TypeError( + 'When setting ' + key + '\n' + 'Setting subscripted must be done using a xarray.DataArray' + ' with the correct dimensions or a constant value ' + '(https://pysd.readthedocs.io/en/master/' + 'getting_started.html)') + + # Try to update stateful component + try: + element = getattr(self.components, stateful_name) + if dims: + value = utils.rearrange( + value, dims, + self._subscript_dict) + element.initialize(value) + modified_statefuls.add(stateful_name) + except NameError: + # Try to override component + raise ValueError( + f"\nUnrecognized stateful '{component_name}'. If you want" + " to set a value of a regular component. Use params={" + f"'{component_name}': {value}" + "} instead.") + + self.clean_caches() + + # get the elements to initialize + elements_to_initialize =\ + self._get_elements_to_initialize(modified_statefuls) + + # Initialize remaining stateful objects + for element_name in self.initialize_order: + if element_name in elements_to_initialize: + self._stateful_elements[element_name].initialize() + + def _get_elements_to_initialize(self, modified_statefuls): + elements_to_initialize = set() + for stateful, deps in self.stateful_initial_dependencies.items(): + if stateful in modified_statefuls: + # if elements initial conditions have been modified + # we should not modify it + continue + for modified_sateteful in modified_statefuls: + if modified_sateteful in deps: + # if element has dependencies on a modified element + # we should re-initialize it + elements_to_initialize.add(stateful) + continue + + return elements_to_initialize + + def set_stateful(self, stateful_dict): + """ + Set stateful values. + + Parameters + ---------- + stateful_dict: dict + Dictionary of the stateful elements and the attributes to change. + + """ + for element, attrs in stateful_dict.items(): + for attr, value in attrs.items(): + setattr(getattr(self.components, element), attr, value) + + def _build_doc(self): + """ + Formats a table of documentation strings to help users remember + variable names, and understand how they are translated into + Python safe names. + + Returns + ------- + docs_df: pandas dataframe + Dataframe with columns for the model components: + - Real names + - Python safe identifiers (as used in model.components) + - Units string + - Documentation strings from the original model file + """ + collector = [] + for name, pyname in self._namespace.items(): + element = getattr(self.components, pyname) + collector.append({ + 'Real Name': name, + 'Py Name': pyname, + 'Subscripts': element.subscripts, + 'Units': element.units, + 'Limits': element.limits, + 'Type': element.type, + 'Subtype': element.subtype, + 'Comment': element.__doc__.strip().strip("\n").strip() + if element.__doc__ else None + }) + + return pd.DataFrame( + collector + ).sort_values(by="Real Name").reset_index(drop=True) + + def __str__(self): + """ Return model source files """ + + # JT: Might be helpful to return not only the source file, but + # also how the instance differs from that source file. This + # would give a more accurate view of the current model. + string = 'Translated Model File: ' + self.py_model_file + if hasattr(self, 'mdl_file'): + string += '\n Original Model File: ' + self.mdl_file + + return string + + +class Model(Macro): + """ + The Model class implements a stateful representation of the system. + It inherits methods from the Macro class to integrate the model and + access and modify model components. It also contains the main + methods for running the model. + + The Model object will be created with components drawn from a + translated Python model file. + + Parameters + ---------- + py_model_file: str or pathlib.Path + Filename of a model which has already been converted into a + Python format. + data_files: dict or list or str or None + The dictionary with keys the name of file and variables to + load the data from there. Or the list of names or name of the + file to search the data in. Only works for TabData type object + and it is neccessary to provide it. Default is None. + initialize: bool + If False, the model will not be initialize when it is loaded. + Default is True. + missing_values : str ("warning", "error", "ignore", "keep") (optional) + What to do with missing values. If "warning" (default) + shows a warning message and interpolates the values. + If "raise" raises an error. If "ignore" interpolates + the values without showing anything. If "keep" it will keep + the missing values, this option may cause the integration to + fail, but it may be used to check the quality of the data. + + """ + def __init__(self, py_model_file, data_files, initialize, missing_values): + """ Sets up the Python objects """ + super().__init__(py_model_file, None, None, Time(), + data_files=data_files) + self.time.stage = 'Load' + self.time.set_control_vars(**self.components._control_vars) + self.data_files = data_files + self.missing_values = missing_values + if initialize: + self.initialize() + + def initialize(self): + """ Initializes the simulation model """ + self.time.stage = 'Initialization' + External.missing = self.missing_values + super().initialize() + + def run(self, params=None, return_columns=None, return_timestamps=None, + initial_condition='original', final_time=None, time_step=None, + saveper=None, reload=False, progress=False, flatten_output=True, + cache_output=True): + """ + Simulate the model's behavior over time. + Return a pandas dataframe with timestamps as rows, + model elements as columns. + + Parameters + ---------- + params: dict (optional) + Keys are strings of model component names. + Values are numeric or pandas Series. + Numeric values represent constants over the model integration. + Timeseries will be interpolated to give time-varying input. + + return_timestamps: list, numeric, ndarray (1D) (optional) + Timestamps in model execution at which to return state information. + Defaults to model-file specified timesteps. + + return_columns: list, 'step' or None (optional) + List of string model component names, returned dataframe + will have corresponding columns. If 'step' only variables with + cache step will be returned. If None, variables with cache step + and run will be returned. Default is None. + + initial_condition: str or (float, dict) (optional) + The starting time, and the state of the system (the values of + all the stocks) at that starting time. 'original' or 'o'uses + model-file specified initial condition. 'current' or 'c' uses + the state of the model after the previous execution. Other str + objects, loads initial conditions from the pickle file with the + given name.(float, dict) tuple lets the user specify a starting + time (float) and (possibly partial) dictionary of initial values + for stock (stateful) objects. Default is 'original'. + + final_time: float or None + Final time of the simulation. If float, the given value will be + used to compute the return_timestamps (if not given) and as a + final time. If None the last value of return_timestamps will be + used as a final time. Default is None. + + time_step: float or None + Time step of the simulation. If float, the given value will be + used to compute the return_timestamps (if not given) and + euler time series. If None the default value from components + will be used. Default is None. + + saveper: float or None + Saving step of the simulation. If float, the given value will be + used to compute the return_timestamps (if not given). If None + the default value from components will be used. Default is None. + + reload : bool (optional) + If True, reloads the model from the translated model file + before making changes. Default is False. + + progress : bool (optional) + If True, a progressbar will be shown during integration. + Default is False. + + flatten_output: bool (optional) + If True, once the output dataframe has been formatted will + split the xarrays in new columns following Vensim's naming + to make a totally flat output. Default is True. + + cache_output: bool (optional) + If True, the number of calls of outputs variables will be increased + in 1. This helps caching output variables if they are called only + once. For performance reasons, if time step = saveper it is + recommended to activate this feature, if time step << saveper + it is recommended to deactivate it. Default is True. + + Examples + -------- + >>> model.run(params={'exogenous_constant': 42}) + >>> model.run(params={'exogenous_variable': timeseries_input}) + >>> model.run(return_timestamps=[1, 2, 3, 4, 10]) + >>> model.run(return_timestamps=10) + >>> model.run(return_timestamps=np.linspace(1, 10, 20)) + + See Also + -------- + pysd.set_components : handles setting model parameters + pysd.set_initial_condition : handles setting initial conditions + + """ + if reload: + self.reload() + + self.progress = progress + + self.time.add_return_timestamps(return_timestamps) + if self.time.return_timestamps is not None and not final_time: + # if not final time given the model will end in the list + # return timestamp (the list is reversed for popping) + if self.time.return_timestamps: + final_time = self.time.return_timestamps[0] + else: + final_time = self.time.next_return + + self.time.set_control_vars( + final_time=final_time, time_step=time_step, saveper=saveper) + + if params: + self.set_components(params) + + # update cache types after setting params + self._assign_cache_type() + + self.set_initial_condition(initial_condition) + + if return_columns is None or isinstance(return_columns, str): + return_columns = self._default_return_columns(return_columns) + + capture_elements, return_addresses = utils.get_return_elements( + return_columns, self._namespace) + + # create a dictionary splitting run cached and others + capture_elements = self._split_capture_elements(capture_elements) + + # include outputs in cache if needed + self._dependencies["OUTPUTS"] = { + element: 1 for element in capture_elements["step"] + } + if cache_output: + self._assign_cache_type() + self._add_constant_cache() + + # Run the model + self.time.stage = 'Run' + # need to clean cache to remove the values from active_initial + self.clean_caches() + + res = self._integrate(capture_elements['step']) + + del self._dependencies["OUTPUTS"] + + self._add_run_elements(res, capture_elements['run']) + self._remove_constant_cache() + + return_df = utils.make_flat_df(res, return_addresses, flatten_output) + + return return_df + + def select_submodel(self, vars=[], modules=[], exogenous_components={}): + """ + Select a submodel from the original model. After selecting a submodel + only the necessary stateful objects for integrating this submodel will + be computed. + + Parameters + ---------- + vars: set or list of strings (optional) + Variables to include in the new submodel. + It can be an empty list if the submodel is only selected by + module names. Default is an empty list. + + modules: set or list of strings (optional) + Modules to include in the new submodel. + It can be an empty list if the submodel is only selected by + variable names. Default is an empty list. Can select a full + module or a submodule by passing the path without the .py, e.g.: + "view_1/submodule1". + + exogenous_components: dictionary of parameters (optional) + Exogenous value to fix to the model variables that are needed + to run the selected submodel. The exogenous_components should + be passed as a dictionary in the same way it is done for + set_components method. By default it is an empty dict and + the needed exogenous components will be set to a numpy.nan value. + + Returns + ------- + None + + Notes + ----- + modules can be only passed when the model has been split in + different files during translation. + + Examples + -------- + >>> model.select_submodel( + ... vars=["Room Temperature", "Teacup temperature"]) + UserWarning: Selecting submodel, to run the full model again use model.reload() + + >>> model.select_submodel( + ... modules=["view_1", "view_2/subview_1"]) + UserWarning: Selecting submodel, to run the full model again use model.reload() + UserWarning: Exogenous components for the following variables are necessary but not given: + initial_value_stock1, stock3 + + >>> model.select_submodel( + ... vars=["stock3"], + ... modules=["view_1", "view_2/subview_1"]) + UserWarning: Selecting submodel, to run the full model again use model.reload() + UserWarning: Exogenous components for the following variables are necessary but not given: + initial_value_stock1, initial_value_stock3 + Please, set them before running the model using set_components method... + + >>> model.select_submodel( + ... vars=["stock3"], + ... modules=["view_1", "view_2/subview_1"], + ... exogenous_components={ + ... "initial_value_stock1": 3, + ... "initial_value_stock3": 5}) + UserWarning: Selecting submodel, to run the full model again use model.reload() + + """ + c_vars, d_vars, s_deps = self._get_dependencies(vars, modules) + warnings.warn( + "Selecting submodel, " + "to run the full model again use model.reload()") + + # get set of all dependencies and all variables to select + all_deps = d_vars["initial"].copy() + all_deps.update(d_vars["step"]) + all_deps.update(d_vars["lookup"]) + + all_vars = all_deps.copy() + all_vars.update(c_vars) + + # clean dependendies and namespace dictionaries, and remove + # the rows from the documentation + for real_name, py_name in self._namespace.copy().items(): + if py_name not in all_vars: + del self._namespace[real_name] + del self._dependencies[py_name] + self._doc.drop( + self._doc.index[self._doc["Real Name"] == real_name], + inplace=True + ) + + for py_name in self._dependencies.copy().keys(): + if py_name.startswith("_") and py_name not in s_deps: + del self._dependencies[py_name] + + # remove active initial from s_deps as they are "fake" objects + # in dependencies + s_deps = { + dep for dep in s_deps if not dep.startswith("_active_initial") + } + + # reassing the dictionary and lists of needed stateful objects + self._stateful_elements = { + name: getattr(self.components, name) + for name in s_deps + if isinstance(getattr(self.components, name), Stateful) + } + self._dynamicstateful_elements = [ + getattr(self.components, name) for name in s_deps + if isinstance(getattr(self.components, name), DynamicStateful) + ] + self._macro_elements = [ + getattr(self.components, name) for name in s_deps + if isinstance(getattr(self.components, name), Macro) + ] + + # keeping only needed external objects + ext_deps = set() + for values in self._dependencies.values(): + if "__external__" in values: + ext_deps.add(values["__external__"]) + self._external_elements = [ + getattr(self.components, name) for name in ext_deps + if isinstance(getattr(self.components, name), External) + ] + + # set all exogenous values to np.nan by default + new_components = {element: np.nan for element in all_deps} + # update exogenous values with the user input + [new_components.update( + { + utils.get_key_and_value_by_insensitive_key_or_value( + key, + self._namespace)[1]: value + }) for key, value in exogenous_components.items()] + + self.set_components(new_components) + + # show a warning message if exogenous values are needed for a + # dependency + new_components = [ + key for key, value in new_components.items() if value is np.nan] + if new_components: + warnings.warn( + "Exogenous components for the following variables are " + f"necessary but not given:\n\t{', '.join(new_components)}" + "\n\n Please, set them before running the model using " + "set_components method...") + + # re-assign the cache_type and initialization order + self._assign_cache_type() + self._get_initialize_order() + + def get_dependencies(self, vars=[], modules=[]): + """ + Get the dependencies of a set of variables or modules. + + Parameters + ---------- + vars: set or list of strings (optional) + Variables to get the dependencies from. + It can be an empty list if the dependencies are computed only + using modules. Default is an empty list. + modules: set or list of strings (optional) + Modules to get the dependencies from. + It can be an empty list if the dependencies are computed only + using variables. Default is an empty list. Can select a full + module or a submodule by passing the path without the .py, e.g.: + "view_1/submodule1". + + Returns + ------- + dependencies: set + Set of dependencies nedded to run vars. + + Notes + ----- + modules can be only passed when the model has been split in + different files during translation. + + Examples + -------- + >>> model.get_dependencies( + ... vars=["Room Temperature", "Teacup temperature"]) + Selected variables (total 1): + room_temperature, teacup_temperature + Stateful objects integrated with the selected variables (total 1): + _integ_teacup_temperature + + >>> model.get_dependencies( + ... modules=["view_1", "view_2/subview_1"]) + Selected variables (total 4): + var1, var2, stock1, delay1 + Dependencies for initialization only (total 1): + initial_value_stock1 + Dependencies that may change over time (total 2): + stock3 + Stateful objects integrated with the selected variables (total 1): + _integ_stock1, _delay_fixed_delay1 + + >>> model.get_dependencies( + ... vars=["stock3"], + ... modules=["view_1", "view_2/subview_1"]) + Selected variables (total 4): + var1, var2, stock1, stock3, delay1 + Dependencies for initialization only (total 1): + initial_value_stock1, initial_value_stock3 + Stateful objects integrated with the selected variables (total 1): + _integ_stock1, _integ_stock3, _delay_fixed_delay1 + + """ + c_vars, d_vars, s_deps = self._get_dependencies(vars, modules) + + text = utils.print_objects_format(c_vars, "Selected variables") + + if d_vars["initial"]: + text += utils.print_objects_format( + d_vars["initial"], + "\nDependencies for initialization only") + if d_vars["step"]: + text += utils.print_objects_format( + d_vars["step"], + "\nDependencies that may change over time") + if d_vars["lookup"]: + text += utils.print_objects_format( + d_vars["lookup"], + "\nLookup table dependencies") + + text += utils.print_objects_format( + s_deps, + "\nStateful objects integrated with the selected variables") + + print(text) + + def _get_dependencies(self, vars=[], modules=[]): + """ + Get the dependencies of a set of variables or modules. + + Parameters + ---------- + vars: set or list of strings (optional) + Variables to get the dependencies from. + It can be an empty list if the dependencies are computed only + using modules. Default is an empty list. + modules: set or list of strings (optional) + Modules to get the dependencies from. + It can be an empty list if the dependencies are computed only + using variables. Default is an empty list. Can select a full + module or a submodule by passing the path without the .py, e.g.: + "view_1/submodule1". + + Returns + ------- + c_vars: set + Set of all selected model variables. + d_deps: dict of sets + Dictionary of dependencies nedded to run vars and modules. + s_deps: set + Set of stateful objects to update when integrating selected + model variables. + + """ + def check_dep(dependencies, initial=False): + for dep in dependencies: + if dep in c_vars or dep.startswith("__"): + pass + elif dep.startswith("_"): + s_deps.add(dep) + dep = self._dependencies[dep] + check_dep(dep["initial"], True) + check_dep(dep["step"]) + else: + if initial and dep not in d_deps["step"]\ + and dep not in d_deps["lookup"]: + d_deps["initial"].add(dep) + else: + if dep in d_deps["initial"]: + d_deps["initial"].remove(dep) + if self.get_args(dep): + d_deps["lookup"].add(dep) + else: + d_deps["step"].add(dep) + + d_deps = {"initial": set(), "step": set(), "lookup": set()} + s_deps = set() + c_vars = {"time", "time_step", "initial_time", "final_time", "saveper"} + for var in vars: + py_name = utils.get_key_and_value_by_insensitive_key_or_value( + var, + self._namespace)[1] + c_vars.add(py_name) + for module in modules: + c_vars.update(self.get_vars_in_module(module)) + + for var in c_vars: + if var == "time": + continue + check_dep(self._dependencies[var]) + + return c_vars, d_deps, s_deps + + def get_vars_in_module(self, module): + """ + Return the name of Python vars in a module. + + Parameters + ---------- + module: str + Name of the module to search in. + + Returns + ------- + vars: set + Set of varible names in the given module. + + """ + if self._modules: + module_content = self._modules.copy() + else: + raise ValueError( + "Trying to get a module from a non-modularized model") + + try: + # get the module or the submodule content + for submodule in module.split("/"): + module_content = module_content[submodule] + module_content = [module_content] + except KeyError: + raise NameError( + f"Module or submodule '{submodule}' not found...\n") + + vars, new_content = set(), [] + + while module_content: + # find the vars in the module or the submodule + for content in module_content: + if isinstance(content, list): + vars.update(content) + else: + [new_content.append(value) for value in content.values()] + + module_content, new_content = new_content, [] + + return vars + + def reload(self): + """ + Reloads the model from the translated model file, so that all the + parameters are back to their original value. + """ + self.__init__(self.py_model_file, data_files=self.data_files, + initialize=True, + missing_values=self.missing_values) + + def _default_return_columns(self, which): + """ + Return a list of the model elements tha change on time that + does not include lookup other functions that take parameters + or run-cached functions. + + Parameters + ---------- + which: str or None + If it is 'step' only cache step elements will be returned. + Else cache 'step' and 'run' elements will be returned. + Default is None. + + Returns + ------- + return_columns: list + List of columns to return + + """ + if which == 'step': + types = ['step'] + else: + types = ['step', 'run'] + + return_columns = [] + + for key, pykey in self._namespace.items(): + if pykey in self.cache_type and self.cache_type[pykey] in types\ + and not self.get_args(pykey): + + return_columns.append(key) + + return return_columns + + def _split_capture_elements(self, capture_elements): + """ + Splits the capture elements list between those with run cache + and others. + + Parameters + ---------- + capture_elements: list + Captured elements list + + Returns + ------- + capture_dict: dict + Dictionary of sets with keywords step and run. + + """ + capture_dict = {'step': set(), 'run': set(), None: set()} + [capture_dict[self.cache_type[element]].add(element) + for element in capture_elements] + return capture_dict + + def set_initial_condition(self, initial_condition): + """ Set the initial conditions of the integration. + + Parameters + ---------- + initial_condition : str or (float, dict) + The starting time, and the state of the system (the values of + all the stocks) at that starting time. 'original' or 'o'uses + model-file specified initial condition. 'current' or 'c' uses + the state of the model after the previous execution. Other str + objects, loads initial conditions from the pickle file with the + given name.(float, dict) tuple lets the user specify a starting + time (float) and (possibly partial) dictionary of initial values + for stock (stateful) objects. + + Examples + -------- + >>> model.set_initial_condition('original') + >>> model.set_initial_condition('current') + >>> model.set_initial_condition('exported_pickle.pic') + >>> model.set_initial_condition((10, {'teacup_temperature': 50})) + + See Also + -------- + model.set_initial_value() + + """ + + if isinstance(initial_condition, tuple): + self.initialize() + self.set_initial_value(*initial_condition) + elif isinstance(initial_condition, str): + if initial_condition.lower() in ["original", "o"]: + self.time.set_control_vars( + initial_time=self.components._control_vars["initial_time"]) + self.initialize() + elif initial_condition.lower() in ["current", "c"]: + pass + else: + self.import_pickle(initial_condition) + else: + raise TypeError( + "Invalid initial conditions. " + + "Check documentation for valid entries or use " + + "'help(model.set_initial_condition)'.") + + def _euler_step(self, dt): + """ + Performs a single step in the euler integration, + updating stateful components + + Parameters + ---------- + dt : float + This is the amount to increase time by this step + + """ + self.state = self.state + self.ddt() * dt + + def _integrate(self, capture_elements): + """ + Performs euler integration. + + Parameters + ---------- + capture_elements: set + Which model elements to capture - uses pysafe names. + + Returns + ------- + outputs: pandas.DataFrame + Output capture_elements data. + + """ + # necessary to have always a non-xaray object for appending objects + # to the DataFrame time will always be a model element and not saved + # TODO: find a better way of saving outputs + capture_elements.add("time") + outputs = pd.DataFrame(columns=capture_elements) + + if self.progress: + # initialize progress bar + progressbar = utils.ProgressBar( + int((self.time.final_time()-self.time())/self.time.time_step()) + ) + else: + # when None is used the update will do nothing + progressbar = utils.ProgressBar(None) + + while self.time.in_bounds(): + if self.time.in_return(): + outputs.at[self.time.round()] = [ + getattr(self.components, key)() + for key in capture_elements] + self._euler_step(self.time.time_step()) + self.time.update(self.time()+self.time.time_step()) + self.clean_caches() + progressbar.update() + + # need to add one more time step, because we run only the state + # updates in the previous loop and thus may be one short. + if self.time.in_return(): + outputs.at[self.time.round()] = [getattr(self.components, key)() + for key in capture_elements] + + progressbar.finish() + + # delete time column as it was created only for avoiding errors + # of appending data. See previous TODO. + del outputs["time"] + return outputs + + def _add_run_elements(self, df, capture_elements): + """ + Adds constant elements to a dataframe. + + Parameters + ---------- + df: pandas.DataFrame + Dataframe to add elements. + + capture_elements: list + List of constant elements + + Returns + ------- + None + + """ + nt = len(df.index.values) + for element in capture_elements: + df[element] = [getattr(self.components, element)()] * nt diff --git a/pysd/py_backend/statefuls.py b/pysd/py_backend/statefuls.py index 6567224e..d7de8846 100644 --- a/pysd/py_backend/statefuls.py +++ b/pysd/py_backend/statefuls.py @@ -1,26 +1,15 @@ """ -The stateful objects are used and updated each time step with an update -method. This include from basic Integ class objects until the Model -class objects. +The Stateful objects are used and updated each time step with an update +method. This include Integs, Delays, Forecasts, Smooths, and Trends, +between others. The Macro class and Model class are also Stateful type. +However, they are defined appart as they are more complex. """ - -import inspect -import re -import pickle import warnings import numpy as np -import pandas as pd import xarray as xr -from . import utils from .functions import zidz, if_then_else -from .external import External, Excels -from .decorators import Cache, constant_cache -from .data import TabData -from .components import Components, Time - -from pysd._version import __version__ small_vensim = 1e-6 # What is considered zero according to Vensim Help @@ -72,20 +61,24 @@ def update(self, state): class Integ(DynamicStateful): """ - Implements INTEG function + Implements INTEG function. + + Parameters + ---------- + ddt: callable + Derivate to integrate. + initial_value: callable + Initial value. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object. Value of the stock. + """ def __init__(self, ddt, initial_value, py_name): - """ - - Parameters - ---------- - ddt: function - This will become an attribute of the object - initial_value: function - Initial value - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.ddt = ddt @@ -107,7 +100,29 @@ def export(self): class Delay(DynamicStateful): """ - Implements DELAY function + Implements DELAY function. + + Parameters + ---------- + delay_input: callable + Input of the delay. + delay_time: callable + Delay time. + initial_value: callable + Initial value. + order: callable + Delay order. + tsetp: callable + The time step of the model. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: numpy.array or xarray.DataArray + Current state of the object. Array of the delays values multiplied + by their corresponding average time. + """ # note that we could have put the `delay_input` argument as a parameter to # the `__call__` function, and more closely mirrored the vensim syntax. @@ -118,17 +133,6 @@ class Delay(DynamicStateful): def __init__(self, delay_input, delay_time, initial_value, order, tstep, py_name): - """ - - Parameters - ---------- - delay_input: function - delay_time: function - initial_value: function - order: function - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.delay_time_func = delay_time @@ -191,7 +195,34 @@ def export(self): class DelayN(DynamicStateful): """ - Implements DELAY N function + Implements DELAY N function. + + Parameters + ---------- + delay_input: callable + Input of the delay. + delay_time: callable + Delay time. + initial_value: callable + Initial value. + order: callable + Delay order. + tsetp: callable + The time step of the model. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: numpy.array or xarray.DataArray + Current state of the object. Array of the delays values multiplied + by their corresponding average time. + + times: numpy.array or xarray.DataArray + Array of delay times used for computing the delay output. + If delay_time is constant, this array will be constant and + DelayN will behave ad Delay. + """ # note that we could have put the `delay_input` argument as a parameter to # the `__call__` function, and more closely mirrored the vensim syntax. @@ -202,17 +233,6 @@ class DelayN(DynamicStateful): def __init__(self, delay_input, delay_time, initial_value, order, tstep, py_name): - """ - - Parameters - ---------- - delay_input: function - delay_time: function - initial_value: function - order: function - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.delay_time_func = delay_time @@ -288,22 +308,34 @@ def export(self): class DelayFixed(DynamicStateful): """ - Implements DELAY FIXED function + Implements DELAY FIXED function. + + Parameters + ---------- + delay_input: callable + Input of the delay. + delay_time: callable + Delay time. + initial_value: callable + Initial value. + tsetp: callable + The time step of the model. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object, equal to pipe[pointer]. + pipe: list + List of the delays values. + pointer: int + Pointer to the last value in the pipe + """ def __init__(self, delay_input, delay_time, initial_value, tstep, py_name): - """ - - Parameters - ---------- - delay_input: function - delay_time: function - initial_value: function - order: function - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.delay_time_func = delay_time @@ -352,33 +384,43 @@ def export(self): class Forecast(DynamicStateful): """ - Implements FORECAST function - """ - def __init__(self, forecast_input, average_time, horizon, py_name): - """ - - Parameters - ---------- - forecast_input: function - average_time: function - horizon: function - py_name: str - Python name to identify the object - """ + Implements FORECAST function. + + Parameters + ---------- + forecast_input: callable + Input of the forecast. + average_time: callable + Average time. + horizon: callable + Forecast horizon. + initial_trend: callable + Initial trend of the forecast. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object. AV value by Vensim docs. + """ + def __init__(self, forecast_input, average_time, horizon, initial_trend, + py_name): super().__init__() self.horizon = horizon self.average_time = average_time self.input = forecast_input + self.initial_trend = initial_trend self.py_name = py_name - def initialize(self, init_val=None): + def initialize(self, init_trend=None): # self.state = AV in the vensim docs - if init_val is None: - self.state = self.input() + if init_trend is None: + self.state = self.input() / (1 + self.initial_trend()) else: - self.state = init_val + self.state = self.input() / (1 + init_trend) if isinstance(self.state, xr.DataArray): self.shape_info = {'dims': self.state.dims, @@ -400,21 +442,30 @@ def export(self): class Smooth(DynamicStateful): """ - Implements SMOOTH function + Implements SMOOTH function. + + Parameters + ---------- + smooth_input: callable + Input of the smooth. + smooth_time: callable + Smooth time. + initial_value: callable + Initial value. + order: callable + Delay order. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: numpy.array or xarray.DataArray + Current state of the object. Array of the inputs having the + value to return in the last position. + """ def __init__(self, smooth_input, smooth_time, initial_value, order, py_name): - """ - - Parameters - ---------- - smooth_input: function - smooth_time: function - initial_value: function - order: function - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.smooth_time_func = smooth_time @@ -461,33 +512,39 @@ def export(self): class Trend(DynamicStateful): """ - Implements TREND function + Implements TREND function. + + Parameters + ---------- + trend_input: callable + Input of the trend. + average_time: callable + Average time. + initial_trend: callable + Initial trend. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object. AV value by Vensim docs. + """ def __init__(self, trend_input, average_time, initial_trend, py_name): - """ - - Parameters - ---------- - trend_input: function - average_time: function - initial_trend: function - py_name: str - Python name to identify the object - """ - super().__init__() self.init_func = initial_trend self.average_time_function = average_time self.input_func = trend_input self.py_name = py_name - def initialize(self, init_val=None): - if init_val is None: + def initialize(self, init_trend=None): + if init_trend is None: self.state = self.input_func()\ / (1 + self.init_func()*self.average_time_function()) else: self.state = self.input_func()\ - / (1 + init_val*self.average_time_function()) + / (1 + init_trend*self.average_time_function()) if isinstance(self.state, xr.DataArray): self.shape_info = {'dims': self.state.dims, @@ -505,17 +562,28 @@ def export(self): class SampleIfTrue(DynamicStateful): + """ + Implements SAMPLE IF TRUE function. + + Parameters + ---------- + condition: callable + Condition for sample. + actual_value: callable + Value to update if condition is true. + initial_value: callable + Initial value. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object. Last actual_value when condition + was true or the initial_value if condition has never been true. + + """ def __init__(self, condition, actual_value, initial_value, py_name): - """ - - Parameters - ---------- - condition: function - actual_value: function - initial_value: function - py_name: str - Python name to identify the object - """ super().__init__() self.condition = condition self.actual_value = actual_value @@ -550,17 +618,22 @@ def export(self): class Initial(Stateful): """ - Implements INITIAL function + Implements INITIAL function. + + Parameters + ---------- + initial_value: callable + Initial value. + py_name: str + Python name to identify the object. + + Attributes + ---------- + state: float or xarray.DataArray + Current state of the object, which will always be the initial_value. + """ def __init__(self, initial_value, py_name): - """ - - Parameters - ---------- - initial_value: function - py_name: str - Python name to identify the object - """ super().__init__() self.init_func = initial_value self.py_name = py_name @@ -573,1595 +646,3 @@ def initialize(self, init_val=None): def export(self): return {'state': self.state} - - -class Macro(DynamicStateful): - """ - The Model class implements a stateful representation of the system, - and contains the majority of methods for accessing and modifying model - components. - - When the instance in question also serves as the root model object - (as opposed to a macro or submodel within another model) it will have - added methods to facilitate execution. - """ - - def __init__(self, py_model_file, params=None, return_func=None, - time=None, time_initialization=None, data_files=None, - py_name=None): - """ - The model object will be created with components drawn from a - translated python model file. - - Parameters - ---------- - py_model_file : - Filename of a model which has already been converted into a - python format. - get_time: - needs to be a function that returns a time object - params - return_func - """ - super().__init__() - self.time = time - self.time_initialization = time_initialization - self.cache = Cache() - self.py_name = py_name - self.external_loaded = False - self.components = Components(str(py_model_file), self.set_components) - - if __version__.split(".")[0]\ - != self.get_pysd_compiler_version().split(".")[0]: - raise ImportError( - "\n\nNot able to import the model. " - + "The model was compiled with a " - + "not compatible version of PySD:" - + "\n\tPySD " + self.get_pysd_compiler_version() - + "\n\nThe current version of PySd is:" - + "\n\tPySD " + __version__ + "\n\n" - + "Please translate again the model with the function" - + " read_vensim or read_xmile.") - - if params is not None: - self.set_components(params, new=True) - for param in params: - self.components._dependencies[ - self.components._namespace[param]] = {"time"} - - # Get the collections of stateful elements and external elements - self._stateful_elements = { - name: getattr(self.components, name) - for name in dir(self.components) - if isinstance(getattr(self.components, name), Stateful) - } - self._dynamicstateful_elements = [ - getattr(self.components, name) for name in dir(self.components) - if isinstance(getattr(self.components, name), DynamicStateful) - ] - self._external_elements = [ - getattr(self.components, name) for name in dir(self.components) - if isinstance(getattr(self.components, name), External) - ] - self._macro_elements = [ - getattr(self.components, name) for name in dir(self.components) - if isinstance(getattr(self.components, name), Macro) - ] - - self._data_elements = [ - getattr(self.components, name) for name in dir(self.components) - if isinstance(getattr(self.components, name), TabData) - ] - - if data_files: - self._get_data(data_files) - - self._assign_cache_type() - self._get_initialize_order() - - if return_func is not None: - self.return_func = getattr(self.components, return_func) - else: - self.return_func = lambda: 0 - - self.py_model_file = str(py_model_file) - - def __call__(self): - return self.return_func() - - def clean_caches(self): - self.cache.clean() - # if nested macros - [macro.clean_caches() for macro in self._macro_elements] - - def _get_data(self, data_files): - if isinstance(data_files, dict): - for data_file, vars in data_files.items(): - for var in vars: - found = False - for element in self._data_elements: - if var in [element.py_name, element.real_name]: - element.load_data(data_file) - found = True - break - if not found: - raise ValueError( - f"'{var}' not found as model data variable") - - else: - for element in self._data_elements: - element.load_data(data_files) - - def _get_initialize_order(self): - """ - Get the initialization order of the stateful elements - and their the full dependencies. - """ - # get the full set of dependencies to initialize an stateful object - # includying all levels - self.stateful_initial_dependencies = { - ext: set() - for ext in self.components._dependencies - if (ext.startswith("_") and not ext.startswith("_active_initial_")) - } - for element in self.stateful_initial_dependencies: - self._get_full_dependencies( - element, self.stateful_initial_dependencies[element], - "initial") - - # get the full dependencies of stateful objects taking into account - # only other objects - current_deps = { - element: [ - dep for dep in deps - if dep in self.stateful_initial_dependencies - ] for element, deps in self.stateful_initial_dependencies.items() - } - - # get initialization order of the stateful elements - self.initialize_order = [] - delete = True - while delete: - delete = [] - for element in current_deps: - if not current_deps[element]: - # if stateful element has no deps on others - # add to the queue to initialize - self.initialize_order.append(element) - delete.append(element) - for element2 in current_deps: - # remove dependency on the initialized element - if element in current_deps[element2]: - current_deps[element2].remove(element) - # delete visited elements - for element in delete: - del current_deps[element] - - if current_deps: - # if current_deps is not an empty set there is a circular - # reference between stateful objects - raise ValueError( - 'Circular initialization...\n' - + 'Not able to initialize the following objects:\n\t' - + '\n\t'.join(current_deps)) - - def _get_full_dependencies(self, element, dep_set, stateful_deps): - """ - Get all dependencies of an element, i.e., also get the dependencies - of the dependencies. When finding an stateful element only dependencies - for initialization are considered. - - Parameters - ---------- - element: str - Element to get the full dependencies. - dep_set: set - Set to include the dependencies of the element. - stateful_deps: "initial" or "step" - The type of dependencies to take in the case of stateful objects. - - Returns - ------- - None - - """ - deps = self.components._dependencies[element] - if element.startswith("_"): - deps = deps[stateful_deps] - for dep in deps: - if dep not in dep_set and not dep.startswith("__")\ - and dep != "time": - dep_set.add(dep) - self._get_full_dependencies(dep, dep_set, stateful_deps) - - def _add_constant_cache(self): - self.constant_funcs = set() - for element, cache_type in self.cache_type.items(): - if cache_type == "run": - if self.get_args(element): - self.components._set_component( - element, - constant_cache(getattr(self.components, element), None) - ) - else: - self.components._set_component( - element, - constant_cache(getattr(self.components, element)) - ) - self.constant_funcs.add(element) - - def _remove_constant_cache(self): - for element in self.constant_funcs: - self.components._set_component( - element, - getattr(self.components, element).function) - self.constant_funcs = set() - - def _assign_cache_type(self): - """ - Assigns the cache type to all the elements from the namespace. - """ - self.cache_type = {"time": None} - - for element in self.components._namespace.values(): - if element not in self.cache_type\ - and element in self.components._dependencies: - self._assign_cache(element) - - for element, cache_type in self.cache_type.items(): - if cache_type is not None: - if element not in self.cache.cached_funcs\ - and self._count_calls(element) > 1: - self.components._set_component( - element, - self.cache(getattr(self.components, element))) - self.cache.cached_funcs.add(element) - - def _count_calls(self, element): - n_calls = 0 - for subelement in self.components._dependencies: - if subelement.startswith("_") and\ - element in self.components._dependencies[subelement]["step"]: - if element in\ - self.components._dependencies[subelement]["initial"]: - n_calls +=\ - 2*self.components._dependencies[subelement][ - "step"][element] - else: - n_calls +=\ - self.components._dependencies[subelement][ - "step"][element] - elif (not subelement.startswith("_") and - element in self.components._dependencies[subelement]): - n_calls +=\ - self.components._dependencies[subelement][element] - - return n_calls - - def _assign_cache(self, element): - """ - Assigns the cache type to the given element and its dependencies if - needed. - - Parameters - ---------- - element: str - Element name. - - Returns - ------- - None - - """ - if not self.components._dependencies[element]: - self.cache_type[element] = "run" - elif "__lookup__" in self.components._dependencies[element]: - self.cache_type[element] = None - elif self._isdynamic(self.components._dependencies[element]): - self.cache_type[element] = "step" - else: - self.cache_type[element] = "run" - for subelement in self.components._dependencies[element]: - if subelement.startswith("_initial_")\ - or subelement.startswith("__"): - continue - if subelement not in self.cache_type: - self._assign_cache(subelement) - if self.cache_type[subelement] == "step": - self.cache_type[element] = "step" - break - - def _isdynamic(self, dependencies): - """ - - Parameters - ---------- - dependencies: iterable - List of dependencies. - - Returns - ------- - isdynamic: bool - True if 'time' or a dynamic stateful objects is in dependencies. - - """ - if "time" in dependencies: - return True - for dep in dependencies: - if dep.startswith("_") and not dep.startswith("_initial_")\ - and not dep.startswith("_active_initial_")\ - and not dep.startswith("__"): - return True - return False - - def get_pysd_compiler_version(self): - """ - Returns the version of pysd complier that used for generating - this model - """ - return self.components.__pysd_version__ - - def initialize(self): - """ - This function initializes the external objects and stateful objects - in the given order. - """ - # Initialize time - if self.time is None: - self.time = self.time_initialization() - - # Reset time to the initial one - self.time.reset() - self.cache.clean() - - self.components._init_outer_references({ - 'scope': self, - 'time': self.time - }) - - if not self.external_loaded: - # Initialize external elements - for element in self._external_elements: - element.initialize() - - # Remove Excel data from memory - Excels.clean() - - self.external_loaded = True - - # Initialize stateful objects - for element_name in self.initialize_order: - self._stateful_elements[element_name].initialize() - - def ddt(self): - return np.array([component.ddt() for component - in self._dynamicstateful_elements], dtype=object) - - @property - def state(self): - return np.array([component.state for component - in self._dynamicstateful_elements], dtype=object) - - @state.setter - def state(self, new_value): - [component.update(val) for component, val - in zip(self._dynamicstateful_elements, new_value)] - - def export(self, file_name): - """ - Export stateful values to pickle file. - - Parameters - ---------- - file_name: str - Name of the file to export the values. - - """ - warnings.warn( - "\nCompatibility of exported states could be broken between" - " different versions of PySD or xarray, current versions:\n" - f"\tPySD {__version__}\n\txarray {xr.__version__}\n" - ) - stateful_elements = { - name: element.export() - for name, element in self._stateful_elements.items() - } - - with open(file_name, 'wb') as file: - pickle.dump( - (self.time(), - stateful_elements, - {'pysd': __version__, 'xarray': xr.__version__} - ), file) - - def import_pickle(self, file_name): - """ - Import stateful values from pickle file. - - Parameters - ---------- - file_name: str - Name of the file to import the values from. - - """ - with open(file_name, 'rb') as file: - time, stateful_dict, metadata = pickle.load(file) - - if __version__ != metadata['pysd']\ - or xr.__version__ != metadata['xarray']: # pragma: no cover - warnings.warn( - "\nCompatibility of exported states could be broken between" - " different versions of PySD or xarray. Current versions:\n" - f"\tPySD {__version__}\n\txarray {xr.__version__}\n" - "Loaded versions:\n" - f"\tPySD {metadata['pysd']}\n\txarray {metadata['xarray']}\n" - ) - - self.set_stateful(stateful_dict) - self.time.set_control_vars(initial_time=time) - - def get_args(self, param): - """ - Returns the arguments of a model element. - - Parameters - ---------- - param: str or func - The model element name or function. - - Returns - ------- - args: list - List of arguments of the function. - - Examples - -------- - >>> model.get_args('birth_rate') - >>> model.get_args('Birth Rate') - - """ - if isinstance(param, str): - func_name = utils.get_key_and_value_by_insensitive_key_or_value( - param, - self.components._namespace)[1] or param - - func = getattr(self.components, func_name) - else: - func = param - - if hasattr(func, 'args'): - # cached functions - return func.args - else: - # regular functions - args = inspect.getfullargspec(func)[0] - if 'self' in args: - args.remove('self') - return args - - def get_coords(self, param): - """ - Returns the coordinates and dims of a model element. - - Parameters - ---------- - param: str or func - The model element name or function. - - Returns - ------- - (coords, dims) or None: (dict, list) or None - The coords and the dimensions of the element if it has. - Otherwise, returns None. - - Examples - -------- - >>> model.get_coords('birth_rate') - >>> model.get_coords('Birth Rate') - - """ - if isinstance(param, str): - func_name = utils.get_key_and_value_by_insensitive_key_or_value( - param, - self.components._namespace)[1] or param - - func = getattr(self.components, func_name) - - else: - func = param - - # TODO simplify this, make all model elements have a dims attribute - if hasattr(func, "dims"): - dims = func.dims - coords = {dim: self.components._subscript_dict[dim] - for dim in dims} - return coords, dims - elif hasattr(func, "state") and isinstance(func.state, xr.DataArray): - value = func() - elif self.get_args(func) and isinstance(func(0), xr.DataArray): - value = func(0) - else: - return None - - dims = list(value.dims) - coords = {coord: list(value.coords[coord].values) - for coord in value.coords} - return coords, dims - - def __getitem__(self, param): - """ - Returns the current value of a model component. - - Parameters - ---------- - param: str or func - The model element name. - - Returns - ------- - value: float or xarray.DataArray - The value of the model component. - - Examples - -------- - >>> model['birth_rate'] - >>> model['Birth Rate'] - - Note - ---- - It will crash if the model component takes arguments. - - """ - func_name = utils.get_key_and_value_by_insensitive_key_or_value( - param, - self.components._namespace)[1] or param - - if self.get_args(getattr(self.components, func_name)): - raise ValueError( - "Trying to get the current value of a lookup " - "to get all the values with the series data use " - "model.get_series_data(param)\n\n") - - return getattr(self.components, func_name)() - - def get_series_data(self, param): - """ - Returns the original values of a model lookup/data component. - - Parameters - ---------- - param: str - The model lookup/data element name. - - Returns - ------- - value: xarray.DataArray - Array with the value of the interpolating series - in the first dimension. - - Examples - -------- - >>> model['room_temperature'] - >>> model['Room temperature'] - - """ - func_name = utils.get_key_and_value_by_insensitive_key_or_value( - param, - self.components._namespace)[1] or param - - try: - if func_name.startswith("_ext_"): - return getattr(self.components, func_name).data - elif self.get_args(getattr(self.components, func_name)): - return getattr(self.components, - "_ext_lookup_" + func_name).data - else: - return getattr(self.components, - "_ext_data_" + func_name).data - except NameError: - raise ValueError( - "Trying to get the values of a hardcoded lookup/data or " - "other type of variable. 'model.get_series_data' only works " - "with external lookups/data objects.\n\n") - - def set_components(self, params, new=False): - """ Set the value of exogenous model elements. - Element values can be passed as keyword=value pairs in the - function call. Values can be numeric type or pandas Series. - Series will be interpolated by integrator. - - Examples - -------- - >>> model.set_components({'birth_rate': 10}) - >>> model.set_components({'Birth Rate': 10}) - - >>> br = pandas.Series(index=range(30), values=np.sin(range(30)) - >>> model.set_components({'birth_rate': br}) - - - """ - # TODO: allow the params argument to take a pandas dataframe, where - # column names are variable names. However some variables may be - # constant or have no values for some index. This should be processed. - # TODO: make this compatible with loading outputs from other files - - for key, value in params.items(): - func_name = utils.get_key_and_value_by_insensitive_key_or_value( - key, - self.components._namespace)[1] - - if isinstance(value, np.ndarray) or isinstance(value, list): - raise TypeError( - 'When setting ' + key + '\n' - 'Setting subscripted must be done using a xarray.DataArray' - ' with the correct dimensions or a constant value ' - '(https://pysd.readthedocs.io/en/master/basic_usage.html)') - - if func_name is None: - raise NameError( - "\n'%s' is not recognized as a model component." - % key) - - if new: - dims, args = None, None - else: - func = getattr(self.components, func_name) - _, dims = self.get_coords(func) or (None, None) - args = self.get_args(func) - - if isinstance(value, pd.Series): - new_function, deps = self._timeseries_component( - value, dims, args) - self.components._dependencies[func_name] = deps - elif callable(value): - new_function = value - args = self.get_args(value) - if args: - # user function needs arguments, add it as a lookup - # to avoud caching it - self.components._dependencies[func_name] =\ - {"__lookup__": None} - else: - # TODO it would be better if we can parse the content - # of the function to get all the dependencies - # user function takes no arguments, using step cache - # adding time as dependency - self.components._dependencies[func_name] = {"time": 1} - - else: - new_function = self._constant_component(value, dims, args) - self.components._dependencies[func_name] = {} - - # this won't handle other statefuls... - if '_integ_' + func_name in dir(self.components): - warnings.warn("Replacing the equation of stock" - + "{} with params".format(key), - stacklevel=2) - - new_function.__name__ = func_name - if dims: - new_function.dims = dims - self.components._set_component(func_name, new_function) - if func_name in self.cache.cached_funcs: - self.cache.cached_funcs.remove(func_name) - - def _timeseries_component(self, series, dims, args=[]): - """ Internal function for creating a timeseries model element """ - # this is only called if the set_component function recognizes a - # pandas series - # TODO: raise a warning if extrapolating from the end of the series. - # TODO: data type variables should be creted using a Data object - # lookup type variables should be created using a Lookup object - if isinstance(series.values[0], xr.DataArray) and args: - # the argument is already given in the model when the model - # is called - return lambda x: utils.rearrange(xr.concat( - series.values, - series.index).interp(concat_dim=x).reset_coords( - 'concat_dim', drop=True), - dims, self.components._subscript_dict), {'__lookup__': None} - - elif isinstance(series.values[0], xr.DataArray): - # the interpolation will be time dependent - return lambda: utils.rearrange(xr.concat( - series.values, - series.index).interp(concat_dim=self.time()).reset_coords( - 'concat_dim', drop=True), - dims, self.components._subscript_dict), {'time': 1} - - elif args and dims: - # the argument is already given in the model when the model - # is called - return lambda x: utils.rearrange( - np.interp(x, series.index, series.values), - dims, self.components._subscript_dict), {'__lookup__': None} - - elif args: - # the argument is already given in the model when the model - # is called - return lambda x:\ - np.interp(x, series.index, series.values), {'__lookup__': None} - - elif dims: - # the interpolation will be time dependent - return lambda: utils.rearrange( - np.interp(self.time(), series.index, series.values), - dims, self.components._subscript_dict), {'time': 1} - - else: - # the interpolation will be time dependent - return lambda:\ - np.interp(self.time(), series.index, series.values),\ - {'time': 1} - - def _constant_component(self, value, dims, args=[]): - """ Internal function for creating a constant model element """ - if args and dims: - # need to pass an argument to keep consistency with the calls - # to the function - return lambda x: utils.rearrange( - value, dims, self.components._subscript_dict) - - elif args: - # need to pass an argument to keep consistency with the calls - # to the function - return lambda x: value - - elif dims: - return lambda: utils.rearrange( - value, dims, self.components._subscript_dict) - - else: - return lambda: value - - def set_initial_value(self, t, initial_value): - """ Set the system initial value. - - Parameters - ---------- - t : numeric - The system time - - initial_value : dict - A (possibly partial) dictionary of the system initial values. - The keys to this dictionary may be either pysafe names or - original model file names - - """ - self.time.set_control_vars(initial_time=t) - stateful_name = "_NONE" - modified_statefuls = set() - - for key, value in initial_value.items(): - component_name =\ - utils.get_key_and_value_by_insensitive_key_or_value( - key, self.components._namespace)[1] - if component_name is not None: - if self.components._dependencies[component_name]: - deps = list(self.components._dependencies[component_name]) - if len(deps) == 1 and deps[0] in self.initialize_order: - stateful_name = deps[0] - else: - component_name = key - stateful_name = key - - try: - _, dims = self.get_coords(component_name) - except TypeError: - dims = None - - if isinstance(value, xr.DataArray)\ - and not set(value.dims).issubset(set(dims)): - raise ValueError( - f"\nInvalid dimensions for {component_name}." - f"It should be a subset of {dims}, " - f"but passed value has {list(value.dims)}") - - if isinstance(value, np.ndarray) or isinstance(value, list): - raise TypeError( - 'When setting ' + key + '\n' - 'Setting subscripted must be done using a xarray.DataArray' - ' with the correct dimensions or a constant value ' - '(https://pysd.readthedocs.io/en/master/basic_usage.html)') - - # Try to update stateful component - try: - element = getattr(self.components, stateful_name) - if dims: - value = utils.rearrange( - value, dims, - self.components._subscript_dict) - element.initialize(value) - modified_statefuls.add(stateful_name) - except NameError: - # Try to override component - raise ValueError( - f"\nUnrecognized stateful '{component_name}'. If you want" - " to set a value of a regular component. Use params={" - f"'{component_name}': {value}" + "} instead.") - - self.clean_caches() - - # get the elements to initialize - elements_to_initialize =\ - self._get_elements_to_initialize(modified_statefuls) - - # Initialize remaining stateful objects - for element_name in self.initialize_order: - if element_name in elements_to_initialize: - self._stateful_elements[element_name].initialize() - - def _get_elements_to_initialize(self, modified_statefuls): - elements_to_initialize = set() - for stateful, deps in self.stateful_initial_dependencies.items(): - if stateful in modified_statefuls: - # if elements initial conditions have been modified - # we should not modify it - continue - for modified_sateteful in modified_statefuls: - if modified_sateteful in deps: - # if element has dependencies on a modified element - # we should re-initialize it - elements_to_initialize.add(stateful) - continue - - return elements_to_initialize - - def set_stateful(self, stateful_dict): - """ - Set stateful values. - - Parameters - ---------- - stateful_dict: dict - Dictionary of the stateful elements and the attributes to change. - - """ - for element, attrs in stateful_dict.items(): - for attr, value in attrs.items(): - setattr(getattr(self.components, element), attr, value) - - def doc(self): - """ - Formats a table of documentation strings to help users remember - variable names, and understand how they are translated into - python safe names. - - Returns - ------- - docs_df: pandas dataframe - Dataframe with columns for the model components: - - Real names - - Python safe identifiers (as used in model.components) - - Units string - - Documentation strings from the original model file - """ - collector = [] - for name, varname in self.components._namespace.items(): - try: - # TODO correct this when Original Eqn is in several lines - docstring = getattr(self.components, varname).__doc__ - lines = docstring.split('\n') - - for unit_line in range(3, 9): - # this loop detects where Units: starts as - # sometimes eqn could be split in several lines - if re.findall('Units:', lines[unit_line]): - break - if unit_line == 3: - eqn = lines[2].replace("Original Eqn:", "").strip() - else: - eqn = '; '.join( - [line.strip() for line in lines[3:unit_line]]) - - collector.append( - {'Real Name': name, - 'Py Name': varname, - 'Eqn': eqn, - 'Unit': lines[unit_line].replace("Units:", "").strip(), - 'Lims': lines[unit_line+1].replace("Limits:", "").strip(), - 'Type': lines[unit_line+2].replace("Type:", "").strip(), - 'Subs': lines[unit_line+3].replace("Subs:", "").strip(), - 'Comment': '\n'.join(lines[(unit_line+4):]).strip()}) - except Exception: - pass - - docs_df = pd.DataFrame(collector) - docs_df.fillna('None', inplace=True) - - order = ['Real Name', 'Py Name', 'Unit', 'Lims', - 'Type', 'Subs', 'Eqn', 'Comment'] - return docs_df[order].sort_values( - by='Real Name').reset_index(drop=True) - - def __str__(self): - """ Return model source files """ - - # JT: Might be helpful to return not only the source file, but - # also how the instance differs from that source file. This - # would give a more accurate view of the current model. - string = 'Translated Model File: ' + self.py_model_file - if hasattr(self, 'mdl_file'): - string += '\n Original Model File: ' + self.mdl_file - - return string - - -class Model(Macro): - def __init__(self, py_model_file, data_files, initialize, missing_values): - """ Sets up the python objects """ - super().__init__(py_model_file, None, None, Time(), - data_files=data_files) - self.time.stage = 'Load' - self.time.set_control_vars(**self.components._control_vars) - self.data_files = data_files - self.missing_values = missing_values - if initialize: - self.initialize() - - def initialize(self): - """ Initializes the simulation model """ - self.time.stage = 'Initialization' - External.missing = self.missing_values - super().initialize() - - def run(self, params=None, return_columns=None, return_timestamps=None, - initial_condition='original', final_time=None, time_step=None, - saveper=None, reload=False, progress=False, flatten_output=False, - cache_output=True): - """ - Simulate the model's behavior over time. - Return a pandas dataframe with timestamps as rows, - model elements as columns. - - Parameters - ---------- - params: dict (optional) - Keys are strings of model component names. - Values are numeric or pandas Series. - Numeric values represent constants over the model integration. - Timeseries will be interpolated to give time-varying input. - - return_timestamps: list, numeric, ndarray (1D) (optional) - Timestamps in model execution at which to return state information. - Defaults to model-file specified timesteps. - - return_columns: list, 'step' or None (optional) - List of string model component names, returned dataframe - will have corresponding columns. If 'step' only variables with - cache step will be returned. If None, variables with cache step - and run will be returned. Default is None. - - initial_condition: str or (float, dict) (optional) - The starting time, and the state of the system (the values of - all the stocks) at that starting time. 'original' or 'o'uses - model-file specified initial condition. 'current' or 'c' uses - the state of the model after the previous execution. Other str - objects, loads initial conditions from the pickle file with the - given name.(float, dict) tuple lets the user specify a starting - time (float) and (possibly partial) dictionary of initial values - for stock (stateful) objects. Default is 'original'. - - final_time: float or None - Final time of the simulation. If float, the given value will be - used to compute the return_timestamps (if not given) and as a - final time. If None the last value of return_timestamps will be - used as a final time. Default is None. - - time_step: float or None - Time step of the simulation. If float, the given value will be - used to compute the return_timestamps (if not given) and - euler time series. If None the default value from components - will be used. Default is None. - - saveper: float or None - Saving step of the simulation. If float, the given value will be - used to compute the return_timestamps (if not given). If None - the default value from components will be used. Default is None. - - reload : bool (optional) - If True, reloads the model from the translated model file - before making changes. Default is False. - - progress : bool (optional) - If True, a progressbar will be shown during integration. - Default is False. - - flatten_output: bool (optional) - If True, once the output dataframe has been formatted will - split the xarrays in new columns following vensim's naming - to make a totally flat output. Default is False. - - cache_output: bool (optional) - If True, the number of calls of outputs variables will be increased - in 1. This helps caching output variables if they are called only - once. For performance reasons, if time step = saveper it is - recommended to activate this feature, if time step << saveper - it is recommended to deactivate it. Default is True. - - Examples - -------- - >>> model.run(params={'exogenous_constant': 42}) - >>> model.run(params={'exogenous_variable': timeseries_input}) - >>> model.run(return_timestamps=[1, 2, 3.1415, 4, 10]) - >>> model.run(return_timestamps=10) - >>> model.run(return_timestamps=np.linspace(1, 10, 20)) - - See Also - -------- - pysd.set_components : handles setting model parameters - pysd.set_initial_condition : handles setting initial conditions - - """ - if reload: - self.reload() - - self.progress = progress - - self.time.add_return_timestamps(return_timestamps) - if self.time.return_timestamps is not None and not final_time: - final_time = self.time.return_timestamps[-1] - - self.time.set_control_vars( - final_time=final_time, time_step=time_step, saveper=saveper) - - if params: - self.set_components(params) - - # update cache types after setting params - self._assign_cache_type() - - self.set_initial_condition(initial_condition) - - if return_columns is None or isinstance(return_columns, str): - return_columns = self._default_return_columns(return_columns) - - capture_elements, return_addresses = utils.get_return_elements( - return_columns, self.components._namespace) - - # create a dictionary splitting run cached and others - capture_elements = self._split_capture_elements(capture_elements) - - # include outputs in cache if needed - self.components._dependencies["OUTPUTS"] = { - element: 1 for element in capture_elements["step"] - } - if cache_output: - self._assign_cache_type() - self._add_constant_cache() - - # Run the model - self.time.stage = 'Run' - # need to clean cache to remove the values from active_initial - self.clean_caches() - - res = self._integrate(capture_elements['step']) - - del self.components._dependencies["OUTPUTS"] - - self._add_run_elements(res, capture_elements['run']) - self._remove_constant_cache() - - return_df = utils.make_flat_df(res, return_addresses, flatten_output) - - return return_df - - def select_submodel(self, vars=[], modules=[], exogenous_components={}): - """ - Select a submodel from the original model. After selecting a submodel - only the necessary stateful objects for integrating this submodel will - be computed. - - Parameters - ---------- - vars: set or list of strings (optional) - Variables to include in the new submodel. - It can be an empty list if the submodel is only selected by - module names. Default is an empty list. - - modules: set or list of strings (optional) - Modules to include in the new submodel. - It can be an empty list if the submodel is only selected by - variable names. Default is an empty list. Can select a full - module or a submodule by passing the path without the .py, e.g.: - "view_1/submodule1". - - exogenous_components: dictionary of parameters (optional) - Exogenous value to fix to the model variables that are needed - to run the selected submodel. The exogenous_components should - be passed as a dictionary in the same way it is done for - set_components method. By default it is an empty dict and - the needed exogenous components will be set to a numpy.nan value. - - Returns - ------- - None - - Notes - ----- - modules can be only passed when the model has been split in - different files during translation. - - Examples - -------- - >>> model.select_submodel( - ... vars=["Room Temperature", "Teacup temperature"]) - UserWarning: Selecting submodel, to run the full model again use model.reload() - - >>> model.select_submodel( - ... modules=["view_1", "view_2/subview_1"]) - UserWarning: Selecting submodel, to run the full model again use model.reload() - UserWarning: Exogenous components for the following variables are necessary but not given: - initial_value_stock1, stock3 - - >>> model.select_submodel( - ... vars=["stock3"], - ... modules=["view_1", "view_2/subview_1"]) - UserWarning: Selecting submodel, to run the full model again use model.reload() - UserWarning: Exogenous components for the following variables are necessary but not given: - initial_value_stock1, initial_value_stock3 - Please, set them before running the model using set_components method... - - >>> model.select_submodel( - ... vars=["stock3"], - ... modules=["view_1", "view_2/subview_1"], - ... exogenous_components={ - ... "initial_value_stock1": 3, - ... "initial_value_stock3": 5}) - UserWarning: Selecting submodel, to run the full model again use model.reload() - - """ - c_vars, d_vars, s_deps = self._get_dependencies(vars, modules) - warnings.warn( - "Selecting submodel, " - "to run the full model again use model.reload()") - - # reassing the dictionary and lists of needed stateful objects - self._stateful_elements = { - name: getattr(self.components, name) - for name in s_deps - if isinstance(getattr(self.components, name), Stateful) - } - self._dynamicstateful_elements = [ - getattr(self.components, name) for name in s_deps - if isinstance(getattr(self.components, name), DynamicStateful) - ] - self._macro_elements = [ - getattr(self.components, name) for name in s_deps - if isinstance(getattr(self.components, name), Macro) - ] - # TODO: include subselection of external objects (update in the deps - # dictionary is needed -> NO BACK COMPATIBILITY) - - # get set of all dependencies and all variables to select - all_deps = d_vars["initial"].copy() - all_deps.update(d_vars["step"]) - all_deps.update(d_vars["lookup"]) - - all_vars = all_deps.copy() - all_vars.update(c_vars) - - # clean dependendies and namespace dictionaries - for real_name, py_name in self.components._namespace.copy().items(): - if py_name not in all_vars: - del self.components._namespace[real_name] - del self.components._dependencies[py_name] - - for py_name in self.components._dependencies.copy().keys(): - if py_name.startswith("_") and py_name not in s_deps: - del self.components._dependencies[py_name] - - # set all exogenous values to np.nan by default - new_components = {element: np.nan for element in all_deps} - # update exogenous values with the user input - [new_components.update( - { - utils.get_key_and_value_by_insensitive_key_or_value( - key, - self.components._namespace)[1]: value - }) for key, value in exogenous_components.items()] - - self.set_components(new_components) - - # show a warning message if exogenous values are needed for a - # dependency - new_components = [ - key for key, value in new_components.items() if value is np.nan] - if new_components: - warnings.warn( - "Exogenous components for the following variables are " - f"necessary but not given:\n\t{', '.join(new_components)}" - "\n\n Please, set them before running the model using " - "set_components method...") - - # re-assign the cache_type and initialization order - self._assign_cache_type() - self._get_initialize_order() - - def get_dependencies(self, vars=[], modules=[]): - """ - Get the dependencies of a set of variables or modules. - - Parameters - ---------- - vars: set or list of strings (optional) - Variables to get the dependencies from. - It can be an empty list if the dependencies are computed only - using modules. Default is an empty list. - modules: set or list of strings (optional) - Modules to get the dependencies from. - It can be an empty list if the dependencies are computed only - using variables. Default is an empty list. Can select a full - module or a submodule by passing the path without the .py, e.g.: - "view_1/submodule1". - - Returns - ------- - dependencies: set - Set of dependencies nedded to run vars. - - Notes - ----- - modules can be only passed when the model has been split in - different files during translation. - - Examples - -------- - >>> model.get_dependencies( - ... vars=["Room Temperature", "Teacup temperature"]) - Selected variables (total 1): - room_temperature, teacup_temperature - Stateful objects integrated with the selected variables (total 1): - _integ_teacup_temperature - - >>> model.get_dependencies( - ... modules=["view_1", "view_2/subview_1"]) - Selected variables (total 4): - var1, var2, stock1, delay1 - Dependencies for initialization only (total 1): - initial_value_stock1 - Dependencies that may change over time (total 2): - stock3 - Stateful objects integrated with the selected variables (total 1): - _integ_stock1, _delay_fixed_delay1 - - >>> model.get_dependencies( - ... vars=["stock3"], - ... modules=["view_1", "view_2/subview_1"]) - Selected variables (total 4): - var1, var2, stock1, stock3, delay1 - Dependencies for initialization only (total 1): - initial_value_stock1, initial_value_stock3 - Stateful objects integrated with the selected variables (total 1): - _integ_stock1, _integ_stock3, _delay_fixed_delay1 - - """ - c_vars, d_vars, s_deps = self._get_dependencies(vars, modules) - - text = utils.print_objects_format(c_vars, "Selected variables") - - if d_vars["initial"]: - text += utils.print_objects_format( - d_vars["initial"], - "\nDependencies for initialization only") - if d_vars["step"]: - text += utils.print_objects_format( - d_vars["step"], - "\nDependencies that may change over time") - if d_vars["lookup"]: - text += utils.print_objects_format( - d_vars["lookup"], - "\nLookup table dependencies") - - text += utils.print_objects_format( - s_deps, - "\nStateful objects integrated with the selected variables") - - print(text) - - def _get_dependencies(self, vars=[], modules=[]): - """ - Get the dependencies of a set of variables or modules. - - Parameters - ---------- - vars: set or list of strings (optional) - Variables to get the dependencies from. - It can be an empty list if the dependencies are computed only - using modules. Default is an empty list. - modules: set or list of strings (optional) - Modules to get the dependencies from. - It can be an empty list if the dependencies are computed only - using variables. Default is an empty list. Can select a full - module or a submodule by passing the path without the .py, e.g.: - "view_1/submodule1". - - Returns - ------- - c_vars: set - Set of all selected model variables. - d_deps: dict of sets - Dictionary of dependencies nedded to run vars and modules. - s_deps: set - Set of stateful objects to update when integrating selected - model variables. - - """ - def check_dep(dependencies, initial=False): - for dep in dependencies: - if dep in c_vars or dep.startswith("__"): - pass - elif dep.startswith("_"): - s_deps.add(dep) - dep = self.components._dependencies[dep] - check_dep(dep["initial"], True) - check_dep(dep["step"]) - else: - if initial and dep not in d_deps["step"]\ - and dep not in d_deps["lookup"]: - d_deps["initial"].add(dep) - else: - if dep in d_deps["initial"]: - d_deps["initial"].remove(dep) - if self.get_args(dep): - d_deps["lookup"].add(dep) - else: - d_deps["step"].add(dep) - - d_deps = {"initial": set(), "step": set(), "lookup": set()} - s_deps = set() - c_vars = {"time", "time_step", "initial_time", "final_time", "saveper"} - for var in vars: - py_name = utils.get_key_and_value_by_insensitive_key_or_value( - var, - self.components._namespace)[1] - c_vars.add(py_name) - for module in modules: - c_vars.update(self.get_vars_in_module(module)) - - for var in c_vars: - if var == "time": - continue - check_dep(self.components._dependencies[var]) - - return c_vars, d_deps, s_deps - - def get_vars_in_module(self, module): - """ - Return the name of python vars in a module. - - Parameters - ---------- - module: str - Name of the module to search in. - - Returns - ------- - vars: set - Set of varible names in the given module. - - """ - try: - module_content = self.components._modules.copy() - except NameError: - raise ValueError( - "Trying to get a module from a non-modularized model") - - try: - # get the module or the submodule content - for submodule in module.split("/"): - module_content = module_content[submodule] - module_content = [module_content] - except KeyError: - raise NameError( - f"Module or submodule '{submodule}' not found...\n") - - vars, new_content = set(), [] - - while module_content: - # find the vars in the module or the submodule - for content in module_content: - if isinstance(content, list): - vars.update(content) - else: - [new_content.append(value) for value in content.values()] - - module_content, new_content = new_content, [] - - return vars - - def reload(self): - """ - Reloads the model from the translated model file, so that all the - parameters are back to their original value. - """ - self.__init__(self.py_model_file, data_files=self.data_files, - initialize=True, - missing_values=self.missing_values) - - def _default_return_columns(self, which): - """ - Return a list of the model elements tha change on time that - does not include lookup other functions that take parameters - or run-cached functions. - - Parameters - ---------- - which: str or None - If it is 'step' only cache step elements will be returned. - Else cache 'step' and 'run' elements will be returned. - Default is None. - - Returns - ------- - return_columns: list - List of columns to return - - """ - if which == 'step': - types = ['step'] - else: - types = ['step', 'run'] - - return_columns = [] - - for key, pykey in self.components._namespace.items(): - if pykey in self.cache_type and self.cache_type[pykey] in types\ - and not self.get_args(pykey): - - return_columns.append(key) - - return return_columns - - def _split_capture_elements(self, capture_elements): - """ - Splits the capture elements list between those with run cache - and others. - - Parameters - ---------- - capture_elements: list - Captured elements list - - Returns - ------- - capture_dict: dict - Dictionary of sets with keywords step and run. - - """ - capture_dict = {'step': set(), 'run': set(), None: set()} - [capture_dict[self.cache_type[element]].add(element) - for element in capture_elements] - return capture_dict - - def set_initial_condition(self, initial_condition): - """ Set the initial conditions of the integration. - - Parameters - ---------- - initial_condition : str or (float, dict) - The starting time, and the state of the system (the values of - all the stocks) at that starting time. 'original' or 'o'uses - model-file specified initial condition. 'current' or 'c' uses - the state of the model after the previous execution. Other str - objects, loads initial conditions from the pickle file with the - given name.(float, dict) tuple lets the user specify a starting - time (float) and (possibly partial) dictionary of initial values - for stock (stateful) objects. - - Examples - -------- - >>> model.set_initial_condition('original') - >>> model.set_initial_condition('current') - >>> model.set_initial_condition('exported_pickle.pic') - >>> model.set_initial_condition((10, {'teacup_temperature': 50})) - - See Also - -------- - model.set_initial_value() - - """ - - if isinstance(initial_condition, tuple): - self.initialize() - self.set_initial_value(*initial_condition) - elif isinstance(initial_condition, str): - if initial_condition.lower() in ["original", "o"]: - self.time.set_control_vars( - initial_time=self.components._control_vars["initial_time"]) - self.initialize() - elif initial_condition.lower() in ["current", "c"]: - pass - else: - self.import_pickle(initial_condition) - else: - raise TypeError( - "Invalid initial conditions. " - + "Check documentation for valid entries or use " - + "'help(model.set_initial_condition)'.") - - def _euler_step(self, dt): - """ - Performs a single step in the euler integration, - updating stateful components - - Parameters - ---------- - dt : float - This is the amount to increase time by this step - - """ - self.state = self.state + self.ddt() * dt - - def _integrate(self, capture_elements): - """ - Performs euler integration. - - Parameters - ---------- - capture_elements: set - Which model elements to capture - uses pysafe names. - - Returns - ------- - outputs: pandas.DataFrame - Output capture_elements data. - - """ - # necessary to have always a non-xaray object for appending objects - # to the DataFrame time will always be a model element and not saved - # TODO: find a better way of saving outputs - capture_elements.add("time") - outputs = pd.DataFrame(columns=capture_elements) - - if self.progress: - # initialize progress bar - progressbar = utils.ProgressBar( - int((self.time.final_time()-self.time())/self.time.time_step()) - ) - else: - # when None is used the update will do nothing - progressbar = utils.ProgressBar(None) - - while self.time.in_bounds(): - if self.time.in_return(): - outputs.at[self.time()] = [getattr(self.components, key)() - for key in capture_elements] - self._euler_step(self.time.time_step()) - self.time.update(self.time()+self.time.time_step()) - self.clean_caches() - progressbar.update() - - # need to add one more time step, because we run only the state - # updates in the previous loop and thus may be one short. - if self.time.in_return(): - outputs.at[self.time()] = [getattr(self.components, key)() - for key in capture_elements] - - progressbar.finish() - - # delete time column as it was created only for avoiding errors - # of appending data. See previous TODO. - del outputs["time"] - return outputs - - def _add_run_elements(self, df, capture_elements): - """ - Adds constant elements to a dataframe. - - Parameters - ---------- - df: pandas.DataFrame - Dataframe to add elements. - - capture_elements: list - List of constant elements - - Returns - ------- - None - - """ - nt = len(df.index.values) - for element in capture_elements: - df[element] = [getattr(self.components, element)()] * nt diff --git a/pysd/py_backend/utils.py b/pysd/py_backend/utils.py index 0dfce6c5..a2d9a787 100644 --- a/pysd/py_backend/utils.py +++ b/pysd/py_backend/utils.py @@ -15,35 +15,6 @@ import pandas as pd -def xrmerge(*das): - """ - Merges xarrays with different dimension sets. - - Parameters - ---------- - *das: xarray.DataArrays - The data arrays to merge. - - - Returns - ------- - da: xarray.DataArray - Merged data array. - - References - ---------- - Thanks to @jcmgray - https://github.com/pydata/xarray/issues/742#issue-130753818 - - In the future, we may not need this as xarray may provide the merge for us. - """ - da = das[0] - for new_da in das[1:]: - da = da.combine_first(new_da) - - return da - - def xrsplit(array): """ Split an array to a list of all the components. @@ -69,7 +40,7 @@ def get_return_elements(return_columns, namespace): """ Takes a list of return elements formatted in vensim's format Varname[Sub1, SUb2] - and returns first the model elements (in python safe language) + and returns first the model elements (in Python safe language) that need to be computed and collected, and secondly the addresses that each element in the return columns list translates to @@ -189,12 +160,14 @@ def _add_flat(savedict, name, values): """ # remove subscripts from name if given name = re.sub(r'\[.*\]', '', name) + dims = values[0].dims + # split values in xarray.DataArray lval = [xrsplit(val) for val in values] for i, ar in enumerate(lval[0]): vals = [float(v[i]) for v in lval] subs = '[' + ','.join([str(ar.coords[dim].values) - for dim in list(ar.coords)]) + ']' + for dim in dims]) + ']' savedict[name+subs] = vals @@ -330,11 +303,11 @@ def load_model_data(root, model_name): """ Used for models split in several files. - Loads subscripts_dic, namespace and modules dictionaries + Loads subscripts and modules dictionaries Parameters ---------- - root: pathlib.Path or str + root: pathlib.Path Path to the model file. model_name: str @@ -342,10 +315,6 @@ def load_model_data(root, model_name): Returns ------- - namespace: dict - Translation from original model element names (keys) to python safe - function identifiers (values). - subscripts: dict Dictionary describing the possible dimensions of the stock's subscripts. @@ -355,25 +324,15 @@ def load_model_data(root, model_name): corresponding variables as values. """ - if isinstance(root, str): # pragma: no cover - # backwards compatibility - # TODO: remove with PySD 3.0.0 - root = Path(root) - with open(root.joinpath("_subscripts_" + model_name + ".json")) as subs: subscripts = json.load(subs) - with open(root.joinpath("_namespace_" + model_name + ".json")) as names: - namespace = json.load(names) - with open(root.joinpath("_dependencies_" + model_name + ".json")) as deps: - dependencies = json.load(deps) - # the _modules.json in the sketch_var folder shows to which module each # variable belongs with open(root.joinpath("modules_" + model_name, "_modules.json")) as mods: modules = json.load(mods) - return namespace, subscripts, dependencies, modules + return subscripts, modules def load_modules(module_name, module_content, work_dir, submodules): @@ -393,7 +352,7 @@ def load_modules(module_name, module_content, work_dir, submodules): module has submodules, whereas if it is a list it means that that particular module/submodule is a final one. - work_dir: str + work_dir: pathlib.Path Path to the module file. submodules: list @@ -408,11 +367,6 @@ def load_modules(module_name, module_content, work_dir, submodules): model file. """ - if isinstance(work_dir, str): # pragma: no cover - # backwards compatibility - # TODO: remove with PySD 3.0.0 - work_dir = Path(work_dir) - if isinstance(module_content, list): with open(work_dir.joinpath(module_name + ".py"), "r", encoding="UTF-8") as mod: diff --git a/pysd/pysd.py b/pysd/pysd.py index 69eaccf9..21b6f93d 100644 --- a/pysd/pysd.py +++ b/pysd/pysd.py @@ -6,7 +6,9 @@ """ import sys -from .py_backend.statefuls import Model + +from pysd.py_backend.model import Model + if sys.version_info[:2] < (3, 7): # pragma: no cover raise RuntimeError( @@ -25,12 +27,12 @@ def read_xmile(xmile_file, data_files=None, initialize=True, missing_values="warning"): """ - Construct a model from `.xmile` file. + Construct a model from a Xmile file. Parameters ---------- - xmile_file : str - The relative path filename for a raw `.xmile` file. + xmile_file: str or pathlib.Path + The relative path filename for a raw Xmile file. initialize: bool (optional) If False, the model will not be initialize when it is loaded. @@ -51,7 +53,7 @@ def read_xmile(xmile_file, data_files=None, initialize=True, Returns ------- model: a PySD class object - Elements from the python model are loaded into the PySD class + Elements from the Python model are loaded into the PySD class and ready to run Examples @@ -59,11 +61,23 @@ def read_xmile(xmile_file, data_files=None, initialize=True, >>> model = read_xmile('../tests/test-models/samples/teacup/teacup.xmile') """ - from .translation.xmile.xmile2py import translate_xmile + from pysd.translators.xmile.xmile_file import XmileFile + from pysd.builders.python.python_model_builder import ModelBuilder + + # Read and parse Xmile file + xmile_file_obj = XmileFile(xmile_file) + xmile_file_obj.parse() + + # get AbstractModel + abs_model = xmile_file_obj.get_abstract_model() - py_model_file = translate_xmile(xmile_file) + # build Python file + py_model_file = ModelBuilder(abs_model).build_model() + + # load Python file model = load(py_model_file, data_files, initialize, missing_values) - model.xmile_file = xmile_file + model.xmile_file = str(xmile_file) + return model @@ -75,7 +89,7 @@ def read_vensim(mdl_file, data_files=None, initialize=True, Parameters ---------- - mdl_file : str + mdl_file: str or pathlib.Path The relative path filename for a raw Vensim `.mdl` file. initialize: bool (optional) @@ -86,7 +100,7 @@ def read_vensim(mdl_file, data_files=None, initialize=True, If given the list of files where the necessary data to run the model is given. Default is None. - missing_values : str ("warning", "error", "ignore", "keep") (optional) + missing_values: str ("warning", "error", "ignore", "keep") (optional) What to do with missing values. If "warning" (default) shows a warning message and interpolates the values. If "raise" raises an error. If "ignore" interpolates @@ -96,7 +110,7 @@ def read_vensim(mdl_file, data_files=None, initialize=True, split_views: bool (optional) If True, the sketch is parsed to detect model elements in each - model view, and then translate each view in a separate python + model view, and then translate each view in a separate Python file. Setting this argument to True is recommended for large models split in many different views. Default is False. @@ -105,18 +119,18 @@ def read_vensim(mdl_file, data_files=None, initialize=True, read from the model, if the encoding is not defined in the model file it will be set to 'UTF-8'. Default is None. + subview_sep: list + Characters used to separate views and subviews (e.g. [",", "."]). + If provided, and split_views=True, each submodule will be placed + inside the directory of the parent view. + **kwargs: (optional) Additional keyword arguments for translation. - subview_sep: list - Characters used to separate views and subviews (e.g. [",", "."]). - If provided, and split_views=True, each submodule will be placed - inside the directory of the parent view. - Returns ------- model: a PySD class object - Elements from the python model are loaded into the PySD class + Elements from the Python model are loaded into the PySD class and ready to run Examples @@ -124,32 +138,49 @@ def read_vensim(mdl_file, data_files=None, initialize=True, >>> model = read_vensim('../tests/test-models/samples/teacup/teacup.mdl') """ - from .translation.vensim.vensim2py import translate_vensim - - py_model_file = translate_vensim(mdl_file, split_views, encoding, **kwargs) + from pysd.translators.vensim.vensim_file import VensimFile + from pysd.builders.python.python_model_builder import ModelBuilder + # Read and parse Vensim file + ven_file = VensimFile(mdl_file, encoding=encoding) + ven_file.parse() + if split_views: + # split variables per views + subview_sep = kwargs.get("subview_sep", "") + ven_file.parse_sketch(subview_sep) + + # get AbstractModel + abs_model = ven_file.get_abstract_model() + + # build Python file + py_model_file = ModelBuilder(abs_model).build_model() + + # load Python file model = load(py_model_file, data_files, initialize, missing_values) model.mdl_file = str(mdl_file) + return model def load(py_model_file, data_files=None, initialize=True, missing_values="warning"): """ - Load a python-converted model file. + Load a Python-converted model file. Parameters ---------- py_model_file : str Filename of a model which has already been converted into a - python format. + Python format. initialize: bool (optional) If False, the model will not be initialize when it is loaded. Default is True. - data_files: list or str or None (optional) - If given the list of files where the necessary data to run the model - is given. Default is None. + data_files: dict or list or str or None + The dictionary with keys the name of file and variables to + load the data from there. Or the list of names or name of the + file to search the data in. Only works for TabData type object + and it is neccessary to provide it. Default is None. missing_values : str ("warning", "error", "ignore", "keep") (optional) What to do with missing values. If "warning" (default) diff --git a/pysd/tools/benchmarking.py b/pysd/tools/benchmarking.py index 8012e35a..51573805 100644 --- a/pysd/tools/benchmarking.py +++ b/pysd/tools/benchmarking.py @@ -2,15 +2,18 @@ Benchmarking tools for testing and comparing outputs between different files. Some of these functions are also used for testing. """ - -import os.path import warnings +from pathlib import Path import numpy as np import pandas as pd -from pysd import read_vensim, read_xmile +from pysd import read_vensim, read_xmile, load from ..py_backend.utils import load_outputs, detect_encoding +from pysd.translators.vensim.vensim_utils import supported_extensions as\ + vensim_extensions +from pysd.translators.xmile.xmile_utils import supported_extensions as\ + xmile_extensions def runner(model_file, canonical_file=None, transpose=False, data_files=None): @@ -40,28 +43,36 @@ def runner(model_file, canonical_file=None, transpose=False, data_files=None): pandas.DataFrame of the model output and the canonical output. """ - directory = os.path.dirname(model_file) + if isinstance(model_file, str): + model_file = Path(model_file) + + directory = model_file.parent # load canonical output if not canonical_file: - if os.path.isfile(os.path.join(directory, 'output.csv')): - canonical_file = os.path.join(directory, 'output.csv') - elif os.path.isfile(os.path.join(directory, 'output.tab')): - canonical_file = os.path.join(directory, 'output.tab') + if directory.joinpath('output.csv').is_file(): + canonical_file = directory.joinpath('output.csv') + elif directory.joinpath('output.tab').is_file(): + canonical_file = directory.joinpath('output.tab') else: - raise FileNotFoundError('\nCanonical output file not found.') + raise FileNotFoundError("\nCanonical output file not found.") canon = load_outputs(canonical_file, transpose=transpose, encoding=detect_encoding(canonical_file)) # load model - if model_file.lower().endswith('.mdl'): + if model_file.suffix.lower() in vensim_extensions: model = read_vensim(model_file, data_files) - elif model_file.lower().endswith(".xmile"): + elif model_file.suffix.lower() in xmile_extensions: model = read_xmile(model_file, data_files) + elif model_file.suffix.lower() == ".py": + model = load(model_file, data_files) else: - raise ValueError('\nModelfile should be *.mdl or *.xmile') + raise ValueError( + "\nThe model file name must be a Vensim" + f" ({', '.join(vensim_extensions)}), a Xmile " + f"({', '.join(xmile_extensions)}) or a PySD (.py) model file...") # run model and return the result @@ -87,8 +98,8 @@ def assert_frames_close(actual, expected, assertion="raise", assertion: str (optional) "raise" if an error should be raised when not able to assert - that two frames are close. Otherwise, it will show a warning - message. Default is "raise". + that two frames are close. If "warning", it will show a warning + message. If "return" it will return information. Default is "raise". verbose: bool (optional) If True, if any column is not close the actual and expected values @@ -102,6 +113,14 @@ def assert_frames_close(actual, expected, assertion="raise", kwargs: Optional rtol and atol values for assert_allclose. + Returns + ------- + (cols, first_false_time, first_false_cols) or None: (set, float, set) or None + If assertion is 'return', return the sets of the all columns that are + different. The time when the first difference was found and the + variables that what different at that time. If assertion is not + 'return' it returns None. + Examples -------- >>> assert_frames_close( @@ -166,15 +185,17 @@ def assert_frames_close(actual, expected, assertion="raise", message = "" if actual_cols.difference(expected_cols): - columns = ["'" + col + "'" for col - in actual_cols.difference(expected_cols)] + columns = sorted([ + "'" + col + "'" for col + in actual_cols.difference(expected_cols)]) columns = ", ".join(columns) message += '\nColumns ' + columns\ + ' from actual values not found in expected values.' if expected_cols.difference(actual_cols): - columns = ["'" + col + "'" for col - in expected_cols.difference(actual_cols)] + columns = sorted([ + "'" + col + "'" for col + in expected_cols.difference(actual_cols)]) columns = ", ".join(columns) message += '\nColumns ' + columns\ + ' from expected values not found in actual values.' @@ -190,8 +211,8 @@ def assert_frames_close(actual, expected, assertion="raise", # TODO let compare dataframes with different timestamps if "warn" assert np.all(np.equal(expected.index.values, actual.index.values)), \ - 'test set and actual set must share a common index' \ - 'instead found' + expected.index.values + 'vs' + actual.index.values + "test set and actual set must share a common index, "\ + "instead found %s vs %s" % (expected.index.values, actual.index.values) # if for Vensim outputs where constant values are only in the first row _remove_constant_nan(expected) @@ -201,13 +222,25 @@ def assert_frames_close(actual, expected, assertion="raise", actual[columns], **kwargs) - if c.all(): - return + if c.all().all(): + return (set(), np.nan, set()) if assertion == "return" else None + + # Get the columns that have the first different value, useful for + # debugging + false_index = c.apply( + lambda x: np.where(~x)[0][0] if not x.all() else np.nan) + index_first_false = int(np.nanmin(false_index)) + time_first_false = c.index[index_first_false] + variable_first_false = sorted( + false_index.index[false_index == index_first_false]) - columns = np.array(columns, dtype=str)[~c.values] + columns = sorted(np.array(columns, dtype=str)[~c.all().values]) assertion_details = "\nFollowing columns are not close:\n\t"\ - + ", ".join(columns) + + ", ".join(columns) + "\n\n"\ + + f"First false values ({time_first_false}):\n\t"\ + + ", ".join(variable_first_false) + if verbose: for col in columns: assertion_details += '\n\n'\ @@ -229,13 +262,15 @@ def assert_frames_close(actual, expected, assertion="raise", if assertion == "raise": raise AssertionError(assertion_details) + elif assertion == "return": + return (set(columns), time_first_false, set(variable_first_false)) else: warnings.warn(assertion_details) def assert_allclose(x, y, rtol=1.e-5, atol=1.e-5): """ - Asserts if all numeric values from two arrays are close. + Asserts if numeric values from two arrays are close. Parameters ---------- @@ -253,7 +288,7 @@ def assert_allclose(x, y, rtol=1.e-5, atol=1.e-5): None """ - return ((abs(x - y) <= atol + rtol * abs(y)) + x.isna()*y.isna()).all() + return ((abs(x - y) <= atol + rtol * abs(y)) + x.isna()*y.isna()) def _remove_constant_nan(df): diff --git a/pysd/translation/builder.py b/pysd/translation/builder.py deleted file mode 100644 index 8be64936..00000000 --- a/pysd/translation/builder.py +++ /dev/null @@ -1,2396 +0,0 @@ -""" -These elements are used by the translator to construct the model from the -interpreted results. It is technically possible to use these functions to -build a model from scratch. But - it would be rather error prone. - -This is code to assemble a pysd model once all of the elements have -been translated from their native language into python compatible syntax. -There should be nothing here that has to know about either vensim or -xmile specific syntax. -""" - -import re -import os.path -import textwrap -import warnings -from io import open -import black -import json - -from . import utils - -from pysd._version import __version__ - - -class Imports(): - """ - Class to save the imported modules information for intelligent import - """ - _numpy, _xarray, _subs = False, False, False - _functions, _statefuls, _external, _data, _utils =\ - set(), set(), set(), set(), set() - _external_libs = {"numpy": "np", "xarray": "xr"} - _internal_libs = [ - "functions", "statefuls", "external", "data", "utils"] - - @classmethod - def add(cls, module, function=None): - """ - Add a function from module. - - Parameters - ---------- - module: str - module name. - - function: str or None - function name. If None module will be set to true. - - """ - if function: - getattr(cls, f"_{module}").add(function) - else: - setattr(cls, f"_{module}", True) - - @classmethod - def get_header(cls, outfile): - """ - Returns the importing information to print in the model file - - Parameters - ---------- - outfile: str - Name of the outfile to print in the header. - - Returns - ------- - text: str - Header of the translated model file. - - """ - text =\ - f'"""\nPython model \'{outfile}\'\nTranslated using PySD\n"""\n\n' - - text += "from pathlib import Path\n" - - for module, shortname in cls._external_libs.items(): - if getattr(cls, f"_{module}"): - text += f"import {module} as {shortname}\n" - - text += "\n" - - for module in cls._internal_libs: - if getattr(cls, f"_{module}"): - text += "from pysd.py_backend.%(module)s import %(methods)s\n"\ - % { - "module": module, - "methods": ", ".join(getattr(cls, f"_{module}"))} - - if cls._subs: - text += "from pysd import subs\n" - - cls.reset() - - return text - - @classmethod - def reset(cls): - """ - Reset the imported modules - """ - cls._numpy, cls._xarray, cls._subs = False, False, False - cls._functions, cls._statefuls, cls._external, cls._data,\ - cls._utils = set(), set(), set(), set(), set() - - -# Variable to save identifiers of external objects -build_names = set() - - -def build_modular_model(elements, subscript_dict, namespace, dependencies, - main_filename, elements_per_view): - - """ - This is equivalent to the build function, but is used when the - split_views parameter is set to True in the read_vensim function. - The main python model file will be named as the original model file, - and stored in the same folder. The modules will be stored in a separate - folder named modules + original_model_name. Three extra json files will - be generated, containing the namespace, subscripts_dict and the module - names plus the variables included in each module, respectively. - - Setting split_views=True is recommended for large models with many - different views. - - Parameters - ---------- - elements: list - Each element is a dictionary, with the various components needed - to assemble a model component in python syntax. This will contain - multiple entries for elements that have multiple definitions in - the original file, and which need to be combined. - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - namespace: dict - Translation from original model element names (keys) to python safe - function identifiers (values). - - main_filename: str - The name of the file to write the main module of the model to. - - elements_per_view: dict - Contains the names of the modules and submodules as keys and the - variables in each specific module inside a list as values. - - """ - root_dir = os.path.dirname(main_filename) - model_name = os.path.basename(main_filename).split(".")[0] - modules_dir = os.path.join(root_dir, "modules_" + model_name) - # create modules directory if it does not exist - os.makedirs(modules_dir, exist_ok=True) - - def process_views_tree(view_name, - view_content, - working_directory, - processed_elements): - """ - Creates a directory tree based on the elements_per_view dictionary. - If it's the final view, it creates a file, if not, it creates a folder. - """ - if isinstance(view_content, list): # will become a module - subview_elems = [] - for element in elements: - if element.get("py_name") in view_content or \ - element.get("parent_name") in view_content: - subview_elems.append(element) - - _build_separate_module(subview_elems, subscript_dict, - view_name, working_directory) - processed_elements += subview_elems - - else: # the current view has subviews - working_directory = os.path.join(working_directory, view_name) - os.makedirs(working_directory, exist_ok=True) - - for subview_name, subview_content in view_content.items(): - process_views_tree(subview_name, - subview_content, - working_directory, - processed_elements) - - processed_elements = [] - for view_name, view_content in elements_per_view.items(): - process_views_tree(view_name, - view_content, - modules_dir, - processed_elements) - - # the unprocessed will go in the main file - unprocessed_elements = [ - element for element in elements if element not in processed_elements - ] - - # building main file using the build function - _build_main_module(unprocessed_elements, subscript_dict, main_filename) - - # create json file for the modules and corresponding model elements - with open(os.path.join(modules_dir, "_modules.json"), "w") as outfile: - json.dump(elements_per_view, outfile, indent=4, sort_keys=True) - - # create single namespace in a separate json file - with open( - os.path.join(root_dir, "_namespace_" + model_name + ".json"), "w" - ) as outfile: - json.dump(namespace, outfile, indent=4, sort_keys=True) - - # create single subscript_dict in a separate json file - with open( - os.path.join(root_dir, "_subscripts_" + model_name + ".json"), "w" - ) as outfile: - json.dump(subscript_dict, outfile, indent=4, sort_keys=True) - - # create single subscript_dict in a separate json file - with open( - os.path.join(root_dir, "_dependencies_" + model_name + ".json"), "w" - ) as outfile: - json.dump(dependencies, outfile, indent=4, sort_keys=True) - - -def _build_main_module(elements, subscript_dict, file_name): - """ - Constructs and writes the python representation of the main model - module, when the split_views=True in the read_vensim function. - - Parameters - ---------- - elements: list - Elements belonging to the main module. Ideally, there should only be - the initial_time, final_time, saveper and time_step, functions, though - there might be others in some situations. Each element is a - dictionary, with the various components needed to assemble a model - component in python syntax. This will contain multiple entries for - elements that have multiple definitions in the original file, and - which need to be combined. - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - file_name: str - Path of the file where the main module will be stored. - - Returns - ------- - None or text: None or str - If file_name="return" it will return the content of the output file - instead of saving it. It is used for testing. - - """ - # separating between control variables and rest of variables - control_vars, funcs = _build_variables(elements, subscript_dict) - - Imports.add("utils", "load_model_data") - Imports.add("utils", "load_modules") - - # import of needed functions and packages - text = Imports.get_header(os.path.basename(file_name)) - - # import namespace from json file - text += textwrap.dedent(""" - __pysd_version__ = '%(version)s' - - __data = { - 'scope': None, - 'time': lambda: 0 - } - - _root = Path(__file__).parent - - _namespace, _subscript_dict, _dependencies, _modules = load_model_data( - _root, "%(outfile)s") - """ % { - "outfile": os.path.basename(file_name).split(".")[0], - "version": __version__ - }) - - text += _get_control_vars(control_vars) - - text += textwrap.dedent(""" - # load modules from modules_%(outfile)s directory - exec(load_modules("modules_%(outfile)s", _modules, _root, [])) - - """ % { - "outfile": os.path.basename(file_name).split(".")[0], - }) - - text += funcs - text = black.format_file_contents(text, fast=True, mode=black.FileMode()) - - # Needed for various sessions - build_names.clear() - - with open(file_name, "w", encoding="UTF-8") as out: - out.write(text) - - -def _build_separate_module(elements, subscript_dict, module_name, module_dir): - """ - Constructs and writes the python representation of a specific model - module, when the split_views=True in the read_vensim function - - Parameters - ---------- - elements: list - Elements belonging to the module module_name. Each element is a - dictionary, with the various components needed to assemble a model - component in python syntax. This will contain multiple entries for - elements that have multiple definitions in the original file, and - which need to be combined. - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - module_name: str - Name of the module - - module_dir: str - Path of the directory where module files will be stored. - - Returns - ------- - None - - """ - text = textwrap.dedent(''' - """ - Module %(module_name)s - Translated using PySD version %(version)s - """ - ''' % { - "module_name": module_name, - "version": __version__, - }) - funcs = _generate_functions(elements, subscript_dict) - text += funcs - text = black.format_file_contents(text, fast=True, mode=black.FileMode()) - - outfile_name = os.path.join(module_dir, module_name + ".py") - - with open(outfile_name, "w", encoding="UTF-8") as out: - out.write(text) - - -def build(elements, subscript_dict, namespace, dependencies, outfile_name): - """ - Constructs and writes the python representation of the model, when the - the split_modules is set to False in the read_vensim function. The entire - model is put in a single python file. - - Parameters - ---------- - elements: list - Each element is a dictionary, with the various components needed to - assemble a model component in python syntax. This will contain - multiple entries for elements that have multiple definitions in the - original file, and which need to be combined. - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - namespace: dict - Translation from original model element names (keys) to python safe - function identifiers (values). - - dependencies: dict - Dependencies dictionary. Variables as keys and set of called values or - objects, objects as keys and a dictionary of dependencies for - initialization and dependencies for run. - - outfile_name: str - The name of the file to write the model to. - - Returns - ------- - None or text: None or str - If outfile_name="return" it will return the content of the output file - instead of saving it. It is used for testing. - - """ - # separating between control variables and rest of variables - control_vars, funcs = _build_variables(elements, subscript_dict) - - text = Imports.get_header(os.path.basename(outfile_name)) - - text += textwrap.dedent(""" - __pysd_version__ = '%(version)s' - - __data = { - 'scope': None, - 'time': lambda: 0 - } - - _root = Path(__file__).parent - - _subscript_dict = %(subscript_dict)s - - _namespace = %(namespace)s - - _dependencies = %(dependencies)s - """ % { - "subscript_dict": repr(subscript_dict), - "namespace": repr(namespace), - "dependencies": repr(dependencies), - "version": __version__, - }) - - text += _get_control_vars(control_vars) + funcs - text = black.format_file_contents(text, fast=True, mode=black.FileMode()) - - # Needed for various sessions - build_names.clear() - - # this is used for testing - if outfile_name == "return": - return text - - with open(outfile_name, "w", encoding="UTF-8") as out: - out.write(text) - - -def _generate_functions(elements, subscript_dict): - """ - Builds all model elements as functions in string format. - NOTE: this function calls the build_element function, which updates the - import_modules. - Therefore, it needs to be executed before the_generate_automatic_imports - function. - - Parameters - ---------- - elements: dict - Each element is a dictionary, with the various components needed to - assemble a model component in python syntax. This will contain - multiple entries for elements that have multiple definitions in the - original file, and which need to be combined. - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - Returns - ------- - funcs: str - String containing all formated model functions - - """ - functions = [build_element(element, subscript_dict) for element in - elements] - - funcs = "%(functions)s" % {"functions": "\n".join(functions)} - funcs = funcs.replace("\t", " ") - - return funcs - - -def _get_control_vars(control_vars): - """ - Create the section of control variables - - Parameters - ---------- - control_vars: str - Functions to define control variables. - - Returns - ------- - text: str - Control variables section and header of model variables section. - - """ - text = textwrap.dedent(""" - ########################################################################## - # CONTROL VARIABLES # - ########################################################################## - %(control_vars_dict)s - def _init_outer_references(data): - for key in data: - __data[key] = data[key] - - - def time(): - return __data['time']() - - """ % {"control_vars_dict": control_vars[0]}) - - text += control_vars[1] - - text += textwrap.dedent(""" - ########################################################################## - # MODEL VARIABLES # - ########################################################################## - """) - - return text - - -def _build_variables(elements, subscript_dict): - """ - Build model variables (functions) and separate then in control variables - and regular variables. - - Parameters - ---------- - elements: list - Model elements. - - subscript_dict: - - Returns - ------- - control_vars, regular_vars: tuple, str - control_vars is a tuple of length 2. First element is the dictionary - of original control vars. Second is the string to add the control - variables' functions. regular_vars is the string to add the regular - variables' functions. - - """ - # returns of the control variables - control_vars_dict = { - "initial_time": ["__data['time'].initial_time()"], - "final_time": ["__data['time'].final_time()"], - "time_step": ["__data['time'].time_step()"], - "saveper": ["__data['time'].saveper()"] - } - regular_vars = [] - control_vars = [] - - for element in elements: - if element["py_name"] in control_vars_dict: - # change the return expression in the element and update the dict - # with the original expression - control_vars_dict[element["py_name"]], element["py_expr"] =\ - element["py_expr"][0], control_vars_dict[element["py_name"]] - control_vars.append(element) - else: - regular_vars.append(element) - - if len(control_vars) == 0: - # macro objects, no control variables - control_vars_dict = "" - else: - control_vars_dict = """ - _control_vars = { - "initial_time": lambda: %(initial_time)s, - "final_time": lambda: %(final_time)s, - "time_step": lambda: %(time_step)s, - "saveper": lambda: %(saveper)s - } - """ % control_vars_dict - - return (control_vars_dict, - _generate_functions(control_vars, subscript_dict)),\ - _generate_functions(regular_vars, subscript_dict) - - -def build_element(element, subscript_dict): - """ - Returns a string that has processed a single element dictionary. - - Parameters - ---------- - element: dict - A dictionary containing at least the elements: - - kind: ['constant', 'setup', 'component', 'lookup'] - Different types of elements will be built differently - - py_expr: str - An expression that has been converted already into python syntax - - subs: list of lists - Each sublist contains coordinates for initialization of a - particular part of a subscripted function, the list of - subscripts vensim attaches to an equation - - subscript_dict: dict - A dictionary containing the names of subscript families (dimensions) - as keys, and a list of the possible positions within that dimension - for each value. - - Returns - ------- - func: str - The function to write in the model file. - - """ - # check the elements with ADD in their name - # as these wones are directly added to the - # external objecets via .add method - py_expr_no_ADD = ["ADD" not in py_expr for py_expr in element["py_expr"]] - - if element["kind"] == "dependencies": - # element only used to update dependencies - return "" - elif sum(py_expr_no_ADD) > 1 and element["kind"] not in [ - "stateful", - "external", - "external_add", - ]: - py_expr_i = [] - # need to append true to the end as the next element is checked - py_expr_no_ADD.append(True) - for i, (py_expr, subs_i) in enumerate(zip(element["py_expr"], - element["subs"])): - if not (py_expr.startswith("xr.") or py_expr.startswith("_ext_")): - # rearrange if it doesn't come from external or xarray - coords = utils.make_coord_dict( - subs_i, - subscript_dict, - terse=False) - coords = { - new_dim: coords[dim] - for new_dim, dim in zip(element["merge_subs"], coords) - } - dims = list(coords) - Imports.add("utils", "rearrange") - py_expr_i.append("rearrange(%s, %s, %s)" % ( - py_expr, dims, coords)) - elif py_expr_no_ADD[i]: - # element comes from external or xarray - py_expr_i.append(py_expr) - Imports.add("utils", "xrmerge") - py_expr = "xrmerge(%s)" % ( - ",\n".join(py_expr_i)) - else: - py_expr = element["py_expr"][0] - - contents = "return %s" % py_expr - - element["subs_dec"] = "" - element["subs_doc"] = "None" - - if element["merge_subs"]: - # We add the list of the subs to the __doc__ of the function - # this will give more information to the user and make possible - # to rewrite subscripted values with model.run(params=X) or - # model.run(initial_condition=(n,x)) - element["subs_doc"] = "%s" % element["merge_subs"] - if element["kind"] in ["component", "setup", "constant", - "component_ext_data", "data"]: - # the decorator is not always necessary as the objects - # defined as xarrays in the model will have the right - # dimensions always, we should try to reduce to the - # maximum when we use it - # re arrange the python object - element["subs_dec"] =\ - "@subs(%s, _subscript_dict)" % element["merge_subs"] - Imports.add("subs") - - indent = 8 - element.update( - { - "ulines": "-" * len(element["real_name"]), - "contents": contents.replace("\n", "\n" + " " * indent), - } - ) - # indent lines 2 onward - - # convert newline indicator and add expected level of indentation - element["doc"] = element["doc"].replace("\\", "\n").replace("\n", "\n ") - - if element["kind"] in ["stateful", "external", "tab_data"]: - func = """ - %(py_name)s = %(py_expr)s - """ % { - "py_name": element["py_name"], - "py_expr": element["py_expr"][0], - } - - elif element["kind"] == "external_add": - # external expressions to be added with .add method - # remove the ADD from the end - py_name = element["py_name"].split("ADD")[0] - func = """ - %(py_name)s%(py_expr)s - """ % { - "py_name": py_name, - "py_expr": element["py_expr"][0], - } - - else: - sep = "\n" + " " * 10 - if len(element["eqn"]) == 1: - # Original equation in the same line - element["eqn"] = element["eqn"][0] - elif len(element["eqn"]) > 5: - # First and last original equations separated by vertical dots - element["eqn"] = ( - sep + element["eqn"][0] + (sep + " .") * 3 + sep - + element["eqn"][-1] - ) - else: - # From 2 to 5 equations in different lines - element["eqn"] = sep + sep.join(element["eqn"]) - - func = ( - ''' - %(subs_dec)s - def %(py_name)s(%(arguments)s): - """ - Real Name: %(real_name)s - Original Eqn: %(eqn)s - Units: %(unit)s - Limits: %(lims)s - Type: %(kind)s - Subs: %(subs_doc)s - - %(doc)s - """ - %(contents)s - ''' - % element - ) - - return textwrap.dedent(func) - - -def merge_partial_elements(element_list): - """ - Merges model elements which collectively all define the model component, - mostly for multidimensional subscripts - - Parameters - ---------- - element_list: list - List of all the elements. - - Returns - ------- - list: - List of merged elements. - - """ - outs = dict() # output data structure - - for element in element_list: - name = element["py_name"] - if name not in outs: - # Use 'expr' for Vensim models, and 'eqn' for Xmile - # (This makes the Vensim equation prettier.) - eqn = element["expr"] if "expr" in element else element["eqn"] - parent_name = element["parent_name"] if "parent_name" in element\ - else None - outs[name] = { - "py_name": element["py_name"], - "real_name": element["real_name"], - "doc": element["doc"], - "py_expr": [element["py_expr"]], # in a list - "unit": element["unit"], - "subs": [element["subs"]], - "merge_subs": element["merge_subs"] - if "merge_subs" in element else None, - "dependencies": element["dependencies"] - if "dependencies" in element else None, - "lims": element["lims"], - "eqn": [eqn.replace(r"\ ", "")], - "parent_name": parent_name, - "kind": element["kind"], - "arguments": element["arguments"], - } - - else: - eqn = element["expr"] if "expr" in element else element["eqn"] - - outs[name]["doc"] = outs[name]["doc"] or element["doc"] - outs[name]["unit"] = outs[name]["unit"] or element["unit"] - outs[name]["lims"] = outs[name]["lims"] or element["lims"] - outs[name]["eqn"] += [eqn.replace(r"\ ", "")] - outs[name]["py_expr"] += [element["py_expr"]] - outs[name]["subs"] += [element["subs"]] - if outs[name]["dependencies"] is not None: - if name.startswith("_"): - # stateful object merge initial and step - for target in outs[name]["dependencies"]: - _merge_dependencies( - outs[name]["dependencies"][target], - element["dependencies"][target]) - else: - # regular element - _merge_dependencies( - outs[name]["dependencies"], - element["dependencies"]) - outs[name]["arguments"] = element["arguments"] - - return list(outs.values()) - - -def _merge_dependencies(current, new): - """ - Merge two dependencies dicts of an element. - - Parameters - ---------- - current: dict - Current dependencies of the element. It will be mutated. - - new: dict - New dependencies to add. - - Returns - ------- - None - - """ - current_set, new_set = set(current), set(new) - for dep in current_set.intersection(new_set): - if dep.startswith("__"): - # if it is special (__lookup__, __external__) continue - continue - # if dependency is in both sum the number of calls - current[dep] += new[dep] - for dep in new_set.difference(current_set): - # if dependency is only in new copy it - current[dep] = new[dep] - - -def build_active_initial_deps(identifier, arguments, deps): - """ - Creates new model element dictionaries for the model elements associated - with a stock. - - Parameters - ---------- - identifier: str - The python-safe name of the stock. - - expression: str - Formula which forms the regular value for active initial. - - initial: str - Formula which forms the initial value for active initial. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - A reference to the gost variable that defines the dependencies. - - new_structure: list - List of additional model element dictionaries. - - """ - deps = build_dependencies( - deps, - { - "initial": [arguments[1]], - "step": [arguments[0]] - }) - - py_name = "_active_initial_%s" % identifier - - # describe the stateful object - new_structure = [{ - "py_name": py_name, - "parent_name": "", - "real_name": "", - "doc": "", - "py_expr": "", - "unit": "", - "lims": "", - "eqn": "", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "dependencies", - "arguments": "", - }] - - return py_name, new_structure - - -def add_stock(identifier, expression, initial_condition, subs, merge_subs, - deps): - """ - Creates new model element dictionaries for the model elements associated - with a stock. - - Parameters - ---------- - identifier: str - The python-safe name of the stock. - - expression: str - The formula which forms the derivative of the stock. - - initial_condition: str - Formula which forms the initial condition for the stock. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - a string to use in place of the 'INTEG...' pieces in the element - expression string, a reference to the stateful object. - - new_structure: list - List of additional model element dictionaries. When there are - subscripts, constructs an external 'init' and 'ddt' function so - that these can be appropriately aggregated. - - """ - Imports.add("statefuls", "Integ") - - deps = build_dependencies( - deps, - { - "initial": [initial_condition], - "step": [expression] - }) - - new_structure = [] - py_name = "_integ_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = "Integ(lambda: %s, lambda: %s, '%s')" % ( - expression, - initial_condition, - py_name, - ) - else: - stateful_py_expr = "Integ(_integ_input_%s, _integ_init_%s, '%s')" % ( - identifier, - identifier, - py_name, - ) - - # following elements not specified in the model file, but must exist - # create the stock initialization element - new_structure.append({ - "py_name": "_integ_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", - "py_expr": initial_condition, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" - % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": "_integ_input_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "component", - "doc": "Provides derivative for %s function" % identifier, - "subs": subs, - "merge_subs": merge_subs, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "py_expr": expression, - "arguments": "", - }) - - # describe the stateful object - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "Representation of %s" % identifier, - "doc": "Integrates Expression %s" % expression, - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_delay(identifier, delay_input, delay_time, initial_value, order, - subs, merge_subs, deps): - """ - Creates code to instantiate a stateful 'Delay' object, - and provides reference to that object's output. - - The name of the stateful object is based upon the passed in parameters, - so if there are multiple places where identical delay functions are - referenced, the translated python file will only maintain one stateful - object, and reference it multiple times. - - Parameters - ---------- - identifier: str - The python-safe name of the delay. - - delay_input: str - Reference to the model component that is the input to the delay. - - delay_time: str - Can be a number (in string format) or a reference to another model - element which will calculate the delay. This is calculated throughout - the simulation at runtime. - - initial_value: str - This is used to initialize the stocks that are present in the delay. - We initialize the stocks with equal values so that the outflow in - the first timestep is equal to this value. - - order: str - The number of stocks in the delay pipeline. As we construct the - delays at build time, this must be an integer and cannot be calculated - from other model components. Anything else will yield a ValueError. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the delay object `__call__` method, which will return - the output of the delay process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Delay") - - deps = build_dependencies( - deps, - { - "initial": [initial_value, order, delay_time], - "step": [delay_time, delay_input] - }) - - new_structure = [] - py_name = "_delay_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = ( - "Delay(lambda: %s, lambda: %s," - "lambda: %s, lambda: %s, time_step, '%s')" - % (delay_input, delay_time, initial_value, order, py_name) - ) - - else: - stateful_py_expr = ( - "Delay(_delay_input_%s, lambda: %s, _delay_init_%s," - "lambda: %s, time_step, '%s')" - % (identifier, delay_time, identifier, order, py_name) - ) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_delay_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": initial_value, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" \ - % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": "_delay_input_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "component", - "doc": "Provides input for %s function" % identifier, - "subs": subs, - "merge_subs": merge_subs, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "py_expr": delay_input, - "arguments": "", - }) - - # describe the stateful object - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "Delay of %s" % delay_input, - "doc": "Delay time: %s \n Delay initial value %s \n Delay order %s" - % (delay_time, initial_value, order), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_delay_f(identifier, delay_input, delay_time, initial_value, deps): - """ - Creates code to instantiate a stateful 'DelayFixed' object, - and provides reference to that object's output. - - The name of the stateful object is based upon the passed in parameters, - so if there are multiple places where identical delay functions are - referenced, the translated python file will only maintain one stateful - object, and reference it multiple times. - - Parameters - ---------- - identifier: str - The python-safe name of the delay. - - delay_input: str - Reference to the model component that is the input to the delay. - - delay_time: str - Can be a number (in string format) or a reference to another model - element which will calculate the delay. This is calculated throughout - the simulation at runtime. - - initial_value: str - This is used to initialize the stocks that are present in the delay. - We initialize the stocks with equal values so that the outflow in - the first timestep is equal to this value. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the delay object `__call__` method, which will return - the output of the delay process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "DelayFixed") - - deps = build_dependencies( - deps, - { - "initial": [initial_value, delay_time], - "step": [delay_input] - }) - - py_name = "_delayfixed_%s" % identifier - - stateful_py_expr = ( - "DelayFixed(lambda: %s, lambda: %s," - "lambda: %s, time_step, '%s')" - % (delay_input, delay_time, initial_value, py_name) - ) - - # describe the stateful object - stateful = { - "py_name": py_name, - "parent_name": identifier, - "real_name": "Delay fixed of %s" % delay_input, - "doc": "DelayFixed time: %s \n Delay initial value %s" - % (delay_time, initial_value), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - } - - return "%s()" % py_name, [stateful] - - -def add_n_delay(identifier, delay_input, delay_time, initial_value, order, - subs, merge_subs, deps): - """ - Creates code to instantiate a stateful 'DelayN' object, - and provides reference to that object's output. - - The name of the stateful object is based upon the passed in parameters, - so if there are multiple places where identical delay functions are - referenced, the translated python file will only maintain one stateful - object, and reference it multiple times. - - Parameters - ---------- - identifier: str - The python-safe name of the delay. - - delay_input: str - Reference to the model component that is the input to the delay. - - delay_time: str - Can be a number (in string format) or a reference to another model - element which will calculate the delay. This is calculated throughout - the simulation at runtime. - - initial_value: str - This is used to initialize the stocks that are present in the delay. - We initialize the stocks with equal values so that the outflow in - the first timestep is equal to this value. - - order: str - The number of stocks in the delay pipeline. As we construct the - delays at build time, this must be an integer and cannot be calculated - from other model components. Anything else will yield a ValueError. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the delay object `__call__` method, which will return - the output of the delay process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "DelayN") - - deps = build_dependencies( - deps, - { - "initial": [initial_value, order, delay_time], - "step": [delay_time, delay_input] - }) - - new_structure = [] - py_name = "_delayn_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = ( - "DelayN(lambda: %s, lambda: %s," - "lambda: %s, lambda: %s, time_step, '%s')" - % (delay_input, delay_time, initial_value, order, py_name) - ) - - else: - stateful_py_expr = ( - "DelayN(_delayn_input_%s, lambda: %s," - " _delayn_init_%s, lambda: %s, time_step, '%s')" - % (identifier, delay_time, identifier, order, py_name) - ) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_delayn_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": initial_value, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" \ - % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": "_delayn_input_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "component", - "doc": "Provides input for %s function" % identifier, - "subs": subs, - "merge_subs": merge_subs, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "py_expr": delay_input, - "arguments": "", - }) - - # describe the stateful object - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "DelayN of %s" % delay_input, - "doc": "DelayN time: %s \n DelayN initial value %s \n DelayN order\ - %s" - % (delay_time, initial_value, order), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_forecast(identifier, forecast_input, average_time, horizon, - subs, merge_subs, deps): - """ - Constructs Forecast object. - - Parameters - ---------- - identifier: str - The python-safe name of the forecast. - - forecast_input: str - Input of the forecast. - - average_time: str - Average time of the forecast. - - horizon: str - Horizon for the forecast. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the forecast object `__call__` method, which will return - the output of the forecast process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Forecast") - - deps = build_dependencies( - deps, - { - "initial": [forecast_input], - "step": [forecast_input, average_time, horizon] - }) - - new_structure = [] - py_name = "_forecast_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = "Forecast(lambda: %s, lambda: %s,"\ - " lambda: %s, '%s')" % ( - forecast_input, average_time, - horizon, py_name) - - else: - # only need to re-dimension init as xarray will take care of other - stateful_py_expr = "Forecast(_forecast_input_%s, lambda: %s,"\ - " lambda: %s, '%s')" % ( - identifier, average_time, - horizon, py_name) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_forecast_input_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": forecast_input, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides input for %s function" - % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "Forecast of %s" % forecast_input, - "doc": "Forecast average time: %s \n Horizon %s" - % (average_time, horizon), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_sample_if_true(identifier, condition, actual_value, initial_value, - subs, merge_subs, deps): - """ - Creates code to instantiate a stateful 'SampleIfTrue' object, - and provides reference to that object's output. - - Parameters - ---------- - identifier: str - The python-safe name of the sample if true. - - condition: str - Reference to another model element that is the condition to the - 'sample if true' function. - - actual_value: str - Can be a number (in string format) or a reference to another model - element which is calculated throughout the simulation at runtime. - - initial_value: str - This is used to initialize the state of the sample if true function. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the sample if true object `__call__` method, - which will return the output of the sample if true process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "SampleIfTrue") - - deps = build_dependencies( - deps, - { - "initial": [initial_value], - "step": [actual_value, condition] - }) - - new_structure = [] - py_name = "_sample_if_true_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = "SampleIfTrue(lambda: %s, lambda: %s,"\ - "lambda: %s, '%s')" % ( - condition, actual_value, initial_value, py_name) - - else: - stateful_py_expr = "SampleIfTrue(lambda: %s, lambda: %s,"\ - "_sample_if_true_init_%s, '%s')" % ( - condition, actual_value, identifier, py_name) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_sample_if_true_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": initial_value, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "" - }) - # describe the stateful object - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "Sample if true of %s" % identifier, - "doc": "Initial value: %s \n Input: %s \n Condition: %s" % ( - initial_value, actual_value, condition), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "" - }) - - return "%s()" % py_name, new_structure - - -def add_n_smooth(identifier, smooth_input, smooth_time, initial_value, order, - subs, merge_subs, deps): - """ - Constructs stock and flow chains that implement the calculation of - a smoothing function. - - Parameters - ---------- - identifier: str - The python-safe name of the smooth. - - smooth_input: str - Reference to the model component that is the input to the - smoothing function. - - smooth_time: str - Can be a number (in string format) or a reference to another model - element which will calculate the delay. This is calculated throughout - the simulation at runtime. - - initial_value: str - This is used to initialize the stocks that are present in the delay. - We initialize the stocks with equal values so that the outflow in - the first timestep is equal to this value. - - order: str - The number of stocks in the delay pipeline. As we construct the delays - at build time, this must be an integer and cannot be calculated from - other model components. Anything else will yield a ValueError. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output - - merge_subs: list of strings - List of the final subscript range of the python array after. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the smooth object `__call__` method, which will return - the output of the smooth process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Smooth") - - deps = build_dependencies( - deps, - { - "initial": [initial_value, order], - "step": [smooth_input, smooth_time] - }) - - new_structure = [] - py_name = "_smooth_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = ( - "Smooth(lambda: %s, lambda: %s," - "lambda: %s, lambda: %s, '%s')" - % (smooth_input, smooth_time, initial_value, order, py_name) - ) - - else: - # only need to re-dimension init and input as xarray will take care of - # other - stateful_py_expr = ( - "Smooth(_smooth_input_%s, lambda: %s," - " _smooth_init_%s, lambda: %s, '%s')" - % (identifier, smooth_time, identifier, order, py_name) - ) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_smooth_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": initial_value, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" % \ - identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": "_smooth_input_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "component", - "doc": "Provides input for %s function" % identifier, - "subs": subs, - "merge_subs": merge_subs, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "py_expr": smooth_input, - "arguments": "", - }) - - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "Smooth of %s" % smooth_input, - "doc": "Smooth time %s \n Initial value %s \n Smooth order %s" % ( - smooth_time, initial_value, order), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_n_trend(identifier, trend_input, average_time, initial_trend, - subs, merge_subs, deps): - """ - Constructs Trend object. - - Parameters - ---------- - identifier: str - The python-safe name of the trend. - - trend_input: str - Input of the trend. - - average_time: str - Average time of the trend. - - trend_initial: str - This is used to initialize the trend. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the trend object `__call__` method, which will return the - output of the trend process. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Trend") - - deps = build_dependencies( - deps, - { - "initial": [initial_trend, trend_input, average_time], - "step": [trend_input, average_time] - }) - - new_structure = [] - py_name = "_trend_%s" % identifier - - if len(subs) == 0: - stateful_py_expr = "Trend(lambda: %s, lambda: %s,"\ - " lambda: %s, '%s')" % ( - trend_input, average_time, - initial_trend, py_name) - - else: - # only need to re-dimension init as xarray will take care of other - stateful_py_expr = "Trend(lambda: %s, lambda: %s,"\ - " _trend_init_%s, '%s')" % ( - trend_input, average_time, - identifier, py_name) - - # following elements not specified in the model file, but must exist - # create the delay initialization element - new_structure.append({ - "py_name": "_trend_init_%s" % identifier, - "parent_name": identifier, - "real_name": "Implicit", - "kind": "setup", # not specified in the model file, but must exist - "py_expr": initial_trend, - "subs": subs, - "merge_subs": merge_subs, - "doc": "Provides initial conditions for %s function" - % identifier, - "unit": "See docs for %s" % identifier, - "lims": "None", - "eqn": "None", - "arguments": "", - }) - - new_structure.append({ - "py_name": py_name, - "parent_name": identifier, - "real_name": "trend of %s" % trend_input, - "doc": "Trend average time: %s \n Trend initial value %s" - % (average_time, initial_trend), - "py_expr": stateful_py_expr, - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - }) - - return "%s()" % py_name, new_structure - - -def add_initial(identifier, value, deps): - """ - Constructs a stateful object for handling vensim's 'Initial' functionality. - - Parameters - ---------- - identifier: str - The python-safe name of the initial. - - value: str - The expression which will be evaluated, and the first value of - which returned. - - deps: dict - The dictionary with all the denpendencies in the expression. - - Returns - ------- - reference: str - Reference to the Initial object `__call__` method, - which will return the first calculated value of `identifier`. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Initial") - - deps = build_dependencies( - deps, - { - "initial": [value] - }) - - py_name = "_initial_%s" % identifier - - stateful = { - "py_name": py_name, - "parent_name": identifier, - "real_name": "Initial %s" % identifier, - "doc": "Returns the value taken on during the initialization phase", - "py_expr": "Initial(lambda: %s, '%s')" % (value, py_name), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - } - - return "%s()" % stateful["py_name"], [stateful] - - -def add_tab_data(identifier, real_name, subs, - subscript_dict, merge_subs, keyword): - """ - Constructs an object for handling Vensim's regular DATA components. - - Parameters - ---------- - identifier: str - The python-safe name of the external values. - - real_name: str - The real name of the variable. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - subscript_dict: dict - Dictionary describing the possible dimensions of the stock's - subscripts. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - keyword: str - Data retrieval method ('interpolate', 'look forward', 'hold backward'). - - Returns - ------- - reference: str - Reference to the TabData object `__call__` method, which will - return the retrieved value of data for the current time step. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("data", "TabData") - - coords = utils.simplify_subscript_input( - utils.make_coord_dict(subs, subscript_dict, terse=False), - subscript_dict, return_full=False, merge_subs=merge_subs) - keyword = ( - "'%s'" % keyword.strip(":").lower() if isinstance(keyword, str) else - keyword) - name = "_data_%s" % identifier - - data = { - "py_name": name, - "parent_name": identifier, - "real_name": "Data for %s" % identifier, - "doc": "Provides data for data variable %s" % identifier, - "py_expr": "TabData('%s', '%s', %s, %s)" % ( - real_name, identifier, coords, keyword), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": subs, - "merge_subs": merge_subs, - "kind": "tab_data", - "arguments": "", - } - - return "%s(time())" % data["py_name"], [data] - - -def add_ext_data(identifier, file_name, tab, time_row_or_col, cell, subs, - subscript_dict, merge_subs, keyword): - """ - Constructs a external object for handling Vensim's GET XLS DATA and - GET DIRECT DATA functionality. - - Parameters - ---------- - identifier: str - The python-safe name of the external values. - - file_name: str - Filepath to the data. - - tab: str - Tab where the data is. - - time_row_or_col: str - Identifier to the starting point of the time dimension. - - cell: str - Cell identifier where the data starts. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - subscript_dict: dict - Dictionary describing the possible dimensions of the stock's - subscripts. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - keyword: str - Data retrieval method ('interpolate', 'look forward', 'hold backward'). - - Returns - ------- - reference: str - Reference to the ExtData object `__call__` method, which will - return the retrieved value of data for the current time step. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("external", "ExtData") - - coords = utils.simplify_subscript_input( - utils.make_coord_dict(subs, subscript_dict, terse=False), - subscript_dict, return_full=False, merge_subs=merge_subs) - keyword = ( - "'%s'" % keyword.strip(":").lower() if isinstance(keyword, str) else - keyword) - name = "_ext_data_%s" % identifier - - # Check if the object already exists - if name in build_names: - # Create a new py_name with ADD_# ending - # This object name will not be used in the model as - # the information is added to the existing object - # with add method. - kind = "external_add" - name = utils.make_add_identifier(name, build_names) - py_expr = ".add(%s, %s, %s, %s, %s, %s)" - else: - # Regular name will be used and a new object will be created - # in the model file. - build_names.add(name) - kind = "external" - py_expr = "ExtData(%s, %s, %s, %s, %s, %s,\n"\ - " _root, '{}')".format(name) - - external = { - "py_name": name, - "parent_name": identifier, - "real_name": "External data for %s" % identifier, - "doc": "Provides data for data variable %s" % identifier, - "py_expr": py_expr % (file_name, tab, time_row_or_col, cell, keyword, - coords), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": subs, - "merge_subs": merge_subs, - "kind": kind, - "arguments": "", - } - - return "%s(time())" % external["py_name"], [external] - - -def add_ext_constant(identifier, file_name, tab, cell, - subs, subscript_dict, merge_subs): - """ - Constructs a external object for handling Vensim's GET XLS CONSTANT and - GET DIRECT CONSTANT functionality. - - Parameters - ---------- - identifier: str - The python-safe name of the external values. - - file_name: str - Filepath to the data. - - tab: str - Tab where the data is. - - cell: str - Cell identifier where the data starts. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - subscript_dict: dict - Dictionary describing the possible dimensions of the stock's - subscripts. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - Returns - ------- - reference: str - Reference to the ExtConstant object `__call__` method, - which will return the read value of the data. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("external", "ExtConstant") - - coords = utils.simplify_subscript_input( - utils.make_coord_dict(subs, subscript_dict, terse=False), - subscript_dict, return_full=False, merge_subs=merge_subs) - name = "_ext_constant_%s" % identifier - - # Check if the object already exists - if name in build_names: - # Create a new py_name with ADD_# ending - # This object name will not be used in the model as - # the information is added to the existing object - # with add method. - kind = "external_add" - name = utils.make_add_identifier(name, build_names) - py_expr = ".add(%s, %s, %s, %s)" - else: - # Regular name will be used and a new object will be created - # in the model file. - kind = "external" - py_expr = "ExtConstant(%s, %s, %s, %s,\n"\ - " _root, '{}')".format(name) - build_names.add(name) - - external = { - "py_name": name, - "parent_name": identifier, - "real_name": "External constant for %s" % identifier, - "doc": "Provides data for constant data variable %s" % identifier, - "py_expr": py_expr % (file_name, tab, cell, coords), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": subs, - "merge_subs": merge_subs, - "kind": kind, - "arguments": "", - } - - return "%s()" % external["py_name"], [external] - - -def add_ext_lookup(identifier, file_name, tab, x_row_or_col, cell, - subs, subscript_dict, merge_subs): - """ - Constructs a external object for handling Vensim's GET XLS LOOKUPS and - GET DIRECT LOOKUPS functionality. - - Parameters - ---------- - identifier: str - The python-safe name of the external values. - - file_name: str - Filepath to the data. - - tab: str - Tab where the data is. - - x_row_or_col: str - Identifier to the starting point of the lookup dimension. - - cell: str - Cell identifier where the data starts. - - subs: list of strings - List of strings of subscript indices that correspond to the - list of expressions, and collectively define the shape of the output. - - subscript_dict: dict - Dictionary describing the possible dimensions of the stock's - subscripts. - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects. - - Returns - ------- - reference: str - Reference to the ExtLookup object `__call__` method, - which will return the retrieved value of data after interpolating it. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("external", "ExtLookup") - - coords = utils.simplify_subscript_input( - utils.make_coord_dict(subs, subscript_dict, terse=False), - subscript_dict, return_full=False, merge_subs=merge_subs) - name = "_ext_lookup_%s" % identifier - - # Check if the object already exists - if name in build_names: - # Create a new py_name with ADD_# ending - # This object name will not be used in the model as - # the information is added to the existing object - # with add method. - kind = "external_add" - name = utils.make_add_identifier(name, build_names) - py_expr = ".add(%s, %s, %s, %s, %s)" - else: - # Regular name will be used and a new object will be created - # in the model file. - kind = "external" - py_expr = "ExtLookup(%s, %s, %s, %s, %s,\n"\ - " _root, '{}')".format(name) - build_names.add(name) - - external = { - "py_name": name, - "parent_name": identifier, - "real_name": "External lookup data for %s" % identifier, - "doc": "Provides data for external lookup variable %s" % identifier, - "py_expr": py_expr % (file_name, tab, x_row_or_col, cell, coords), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": subs, - "merge_subs": merge_subs, - "kind": kind, - "arguments": "x", - } - - return "%s(x)" % external["py_name"], [external] - - -def add_macro(identifier, macro_name, filename, arg_names, arg_vals, deps): - """ - Constructs a stateful object instantiating a 'Macro'. - - Parameters - ---------- - identifier: str - The python-safe name of the element that calls the macro. - - macro_name: str - Python safe name for macro. - - filename: str - Filepath to macro definition. - - func_args: dict - Dictionary of values to be passed to macro, {key: function}. - - Returns - ------- - reference: str - Reference to the Initial object `__call__` method, - which will return the first calculated value of `initial_input`. - - new_structure: list - List of element construction dictionaries for the builder to assemble. - - """ - Imports.add("statefuls", "Macro") - - deps = build_dependencies( - deps, - { - "initial": arg_vals, - "step": arg_vals - }) - - py_name = "_macro_" + macro_name + "_" + identifier - - func_args = "{ %s }" % ", ".join( - ["'%s': lambda: %s" % (key, val) for key, val in zip(arg_names, - arg_vals)]) - - stateful = { - "py_name": py_name, - "parent_name": identifier, - "real_name": "Macro Instantiation of " + macro_name, - "doc": "Instantiates the Macro", - "py_expr": "Macro(_root.joinpath('%s'), %s, '%s'," - " time_initialization=lambda: __data['time']," - " py_name='%s')" % (filename, func_args, macro_name, py_name), - "unit": "None", - "lims": "None", - "eqn": "None", - "subs": "", - "merge_subs": None, - "dependencies": deps, - "kind": "stateful", - "arguments": "", - } - - return "%s()" % stateful["py_name"], [stateful] - - -def add_incomplete(var_name, dependencies): - """ - Incomplete functions don't really need to be 'builders' as they - add no new real structure, but it's helpful to have a function - in which we can raise a warning about the incomplete equation - at translate time. - - parameters - ---------- - var_name: str - The python-safe name of the incomplete variable. - - dependencies: list - The list of the dependencies in the variable. - - Returns - ------- - str: - Inclompete funcion call. - - """ - Imports.add("functions", "incomplete") - - warnings.warn( - "%s has no equation specified" % var_name, SyntaxWarning, stacklevel=2 - ) - - # first arg is `self` reference - return "incomplete(%s)" % ", ".join(dependencies), [] - - -def build_dependencies(deps, exps): - # TODO document - - deps_dict = {"initial": {}, "step": {}} - - for target, exprs in exps.items(): - expr = ".".join(exprs) - for dep in deps: - n_calls = len( - re.findall( - "(? 0: - deps_dict[target][dep] = n_calls - - return deps_dict - - -def build_function_call(function_def, user_arguments, dependencies=set()): - """ - Build a function call using the arguments from the original model. - - Parameters - ---------- - function_def: dict - Function definition map with following keys: - - name: name of the function. - - parameters: list with description of all parameters of this function - - name - - optional? - - type: [ - "expression", - provide converted expression as parameter for - runtime evaluating before the method call - "lambda", - provide lambda expression as parameter for - delayed runtime evaluation in the method call - "time", - provide access to current instance of - time object - "scope", - provide access to current instance of - scope object (instance of Macro object) - "predef" - provide an invariant argument. Argument not - given in Vensim/Xmile but needed for python. - "ignore" - ignore an user argument. Argument given in - Vensim/Xmile but not needed for python. - "subs_range_to_list" - - provides the list of subscripts in a given - subscript range - ] - - user_arguments: list - Arguments provided from model. - - dependencies: set (optional) - Set to update dependencies if needed. - - Returns - ------- - str: - Function call. - - """ - if isinstance(function_def, str): - return function_def + "(" + ",".join(user_arguments) + ")" - - if function_def["name"] == "not_implemented_function": - user_arguments = ["'" + function_def["original_name"] + "'"] + \ - user_arguments - warnings.warn( - "\n\nTrying to translate " - + function_def["original_name"] - + " which it is not implemented on PySD. The translated " - + "model will crash... " - ) - - if "module" in function_def: - if function_def["module"] in ["numpy", "xarray"]: - # import external modules - Imports.add(function_def["module"]) - else: - # import method from PySD module - Imports.add(function_def["module"], function_def["name"]) - - if "parameters" in function_def: - parameters = function_def["parameters"] - user_argument = "" - arguments = [] - argument_idx = 0 - for parameter_idx in range(len(parameters)): - parameter_def = parameters[parameter_idx] - is_optional = ( - parameter_def["optional"] if "optional" in parameter_def else - False - ) - if argument_idx >= len(user_arguments) and is_optional: - break - - parameter_type = ( - parameter_def["type"] if "type" in parameter_def else - "expression") - - if parameter_type in ["expression", - "lambda", - "subs_range_to_list"]: - user_argument = user_arguments[argument_idx] - argument_idx += 1 - elif parameter_type == "time": - if "time" in dependencies: - dependencies["time"] += 1 - else: - dependencies["time"] = 1 - elif parameter_type == "ignore": - argument_idx += 1 - continue - - arguments.append( - { - "expression": user_argument, - "lambda": "lambda: " + user_argument, - "time": "__data['time']", - "scope": "__data['scope']", - "predef": parameter_def["name"], - "subs_range_to_list": f"_subscript_dict['{user_argument}']" - }[parameter_type] - ) - - return function_def["name"] + "(" + ", ".join(arguments) + ")" - - return function_def["name"] + "(" + ",".join(user_arguments) + ")" diff --git a/pysd/translation/utils.py b/pysd/translation/utils.py deleted file mode 100644 index 601926f6..00000000 --- a/pysd/translation/utils.py +++ /dev/null @@ -1,522 +0,0 @@ -""" -These are general utilities used by the builder.py, functions.py or the -model file. Vensim's function equivalents should not go here but in -functions.py -""" - -import warnings -from collections.abc import Mapping - -import regex as re -import numpy as np - -# used to create python safe names with the variable reserved_words -from keyword import kwlist -from builtins import __dir__ as bidir -from ..py_backend.components import __dir__ as cdir -from ..py_backend.data import __dir__ as ddir -from ..py_backend.decorators import __dir__ as dedir -from ..py_backend.external import __dir__ as edir -from ..py_backend.functions import __dir__ as fdir -from ..py_backend.statefuls import __dir__ as sdir -from ..py_backend.utils import __dir__ as udir - - -reserved_words = set( - dir() + bidir() + cdir() + ddir() + dedir() + edir() + fdir() - + sdir() + udir()) -reserved_words = reserved_words.union(kwlist) - - -def find_subscript_name(subscript_dict, element, avoid=[]): - """ - Given a subscript dictionary, and a member of a subscript family, - return the first key of which the member is within the value list. - If element is already a subscript name, return that. - - Parameters - ---------- - subscript_dict: dict - Follows the {'subscript name':['list','of','subscript','elements']} - format. - - element: str - - avoid: list (optional) - List of subscripts to avoid. Default is an empty list. - - Returns - ------- - - Examples - -------- - >>> find_subscript_name({'Dim1': ['A', 'B'], - ... 'Dim2': ['C', 'D', 'E'], - ... 'Dim3': ['F', 'G', 'H', 'I']}, - ... 'D') - 'Dim2' - >>> find_subscript_name({'Dim1': ['A', 'B'], - ... 'Dim2': ['A', 'B'], - ... 'Dim3': ['A', 'B']}, - ... 'B') - 'Dim1' - >>> find_subscript_name({'Dim1': ['A', 'B'], - ... 'Dim2': ['A', 'B'], - ... 'Dim3': ['A', 'B']}, - ... 'B', - ... avoid=['Dim1']) - 'Dim2' - """ - if element in subscript_dict.keys(): - return element - - for name, elements in subscript_dict.items(): - if element in elements and name not in avoid: - return name - - -def make_coord_dict(subs, subscript_dict, terse=True): - """ - This is for assisting with the lookup of a particular element, such that - the output of this function would take the place of %s in this expression. - - `variable.loc[%s]` - - Parameters - ---------- - subs: list of strings - coordinates, either as names of dimensions, or positions within - a dimension. - - subscript_dict: dict - the full dictionary of subscript names and values. - - terse: bool (optional) - If True, includes only elements that do not cover the full range of - values in their respective dimension.If False, returns all dimensions. - Default is True. - - Returns - ------- - coordinates: dict - Coordinates needed to access the xarray quantities we're interested in. - - Examples - -------- - >>> make_coord_dict(['Dim1', 'D'], {'Dim1': ['A', 'B', 'C'], - ... 'Dim2': ['D', 'E', 'F']}) - {'Dim2': ['D']} - >>> make_coord_dict(['Dim1', 'D'], {'Dim1': ['A', 'B', 'C'], - ... 'Dim2':['D', 'E', 'F']}, terse=False) - {'Dim2': ['D'], 'Dim1': ['A', 'B', 'C']} - - """ - sub_elems_list = [y for x in subscript_dict.values() for y in x] - coordinates = {} - for sub in subs: - if sub in sub_elems_list: - name = find_subscript_name(subscript_dict, sub, avoid=subs) - coordinates[name] = [sub] - elif not terse: - coordinates[sub] = subscript_dict[sub] - return coordinates - - -def make_merge_list(subs_list, subscript_dict, element=""): - """ - This is for assisting when building xrmerge. From a list of subscript - lists returns the final subscript list after mergin. Necessary when - merging variables with subscripts comming from different definitions. - - Parameters - ---------- - subs_list: list of lists of strings - Coordinates, either as names of dimensions, or positions within - a dimension. - - subscript_dict: dict - The full dictionary of subscript names and values. - - element: str (optional) - Element name, if given it will be printed with any error or - warning message. Default is "". - - Returns - ------- - dims: list - Final subscripts after merging. - - Examples - -------- - >>> make_merge_list([['upper'], ['C']], {'all': ['A', 'B', 'C'], - ... 'upper': ['A', 'B']}) - ['all'] - - """ - coords_set = [set() for i in range(len(subs_list[0]))] - coords_list = [ - make_coord_dict(subs, subscript_dict, terse=False) - for subs in subs_list - ] - - # update coords set - [[coords_set[i].update(coords[dim]) for i, dim in enumerate(coords)] - for coords in coords_list] - - dims = [None] * len(coords_set) - # create an array with the name of the subranges for all merging elements - dims_list = np.array([list(coords) for coords in coords_list]).transpose() - indexes = np.arange(len(dims)) - - for i, coord2 in enumerate(coords_set): - dims1 = [ - dim for dim in dims_list[i] - if dim is not None and set(subscript_dict[dim]) == coord2 - ] - if dims1: - # if the given coordinate already matches return it - dims[i] = dims1[0] - else: - # find a suitable coordinate - other_dims = dims_list[indexes != i] - for name, elements in subscript_dict.items(): - if coord2 == set(elements) and name not in other_dims: - dims[i] = name - break - - if not dims[i]: - # the dimension is incomplete use the smaller - # dimension that completes it - for name, elements in subscript_dict.items(): - if coord2.issubset(set(elements))\ - and name not in other_dims: - dims[i] = name - warnings.warn( - element - + "\nDimension given by subscripts:" - + "\n\t{}\nis incomplete ".format(coord2) - + "using {} instead.".format(name) - + "\nSubscript_dict:" - + "\n\t{}".format(subscript_dict) - ) - break - - if not dims[i]: - for name, elements in subscript_dict.items(): - if coord2 == set(elements): - j = 1 - while name + str(j) in subscript_dict.keys(): - j += 1 - subscript_dict[name + str(j)] = elements - dims[i] = name + str(j) - warnings.warn( - element - + "\nAdding new subscript range to" - + " subscript_dict:\n" - + name + str(j) + ": " + ', '.join(elements)) - break - - if not dims[i]: - # not able to find the correct dimension - raise ValueError( - element - + "\nImpossible to find the dimension that contains:" - + "\n\t{}\nFor subscript_dict:".format(coord2) - + "\n\t{}".format(subscript_dict) - ) - - return dims - - -def make_python_identifier(string, namespace=None): - """ - Takes an arbitrary string and creates a valid Python identifier. - - If the input string is in the namespace, return its value. - - If the python identifier created is already in the namespace, - but the input string is not (ie, two similar strings resolve to - the same python identifier) - - or if the identifier is a reserved word in the reserved_words - list, or is a python default reserved word, - adds _1, or if _1 is in the namespace, _2, etc. - - Parameters - ---------- - string: str - The text to be converted into a valid python identifier. - - namespace: dict - Map of existing translations into python safe identifiers. - This is to ensure that two strings are not translated into - the same python identifier. If string is already in the namespace - its value will be returned. Otherwise, namespace will be mutated - adding string as a new key and its value. - - Returns - ------- - identifier: str - A vaild python identifier based on the input string. - - Examples - -------- - >>> make_python_identifier('Capital') - 'capital' - - >>> make_python_identifier('multiple words') - 'multiple_words' - - >>> make_python_identifier('multiple spaces') - 'multiple_spaces' - - When the name is a python keyword, add '_1' to differentiate it - >>> make_python_identifier('for') - 'for_1' - - Remove leading and trailing whitespace - >>> make_python_identifier(' whitespace ') - 'whitespace' - - Remove most special characters outright: - >>> make_python_identifier('H@t tr!ck') - 'ht_trck' - - remove leading digits - >>> make_python_identifier('123abc') - 'nvs_123abc' - - already in namespace - >>> make_python_identifier('Var$', namespace={'Var$': 'var'}) - ''var' - - namespace conflicts - >>> make_python_identifier('Var@', namespace={'Var$': 'var'}) - 'var_1' - - >>> make_python_identifier('Var$', namespace={'Var@': 'var', - ... 'Var%':'var_1'}) - 'var_2' - - References - ---------- - Identifiers must follow the convention outlined here: - https://docs.python.org/2/reference/lexical_analysis.html#identifiers - - """ - if namespace is None: - namespace = dict() - - if string in namespace: - return namespace[string] - - # create a working copy (and make it lowercase, while we're at it) - s = string.lower() - - # remove leading and trailing whitespace - s = s.strip() - - # Make spaces into underscores - s = re.sub(r"[\s\t\n]+", "_", s) - - # Remove invalid characters - s = re.sub(r"[^\p{l}\p{m}\p{n}_]", "", s) - - # If leading character is not a letter add nvs_. - # Only letters can be leading characters. - if re.findall(r"^[^\p{l}_]", s): - s = "nvs_" + s - elif re.findall(r"^_", s): - s = "nvs" + s - - # reserved the names of PySD functions and methods and other vars - # in the namespace - used_words = reserved_words.union(namespace.values()) - - # Check that the string is not a python identifier - identifier = s - i = 1 - while identifier in used_words: - identifier = s + '_' + str(i) - i += 1 - - namespace[string] = identifier - - return identifier - - -def make_add_identifier(identifier, build_names): - """ - Takes an existing used Python identifier and attatch a unique - identifier with ADD_# ending. - - Used for add new information to an existing external object. - build_names will be updated inside this functions as a set - is mutable. - - Parameters - ---------- - identifier: str - Existing python identifier. - - build_names: set - Set of the already used identifiers for external objects. - - Returns - ------- - identifier: str - A vaild python identifier based on the input indentifier - and the existing ones. - - """ - identifier += "ADD_" - number = 1 - # iterate until finding a non-used identifier - while identifier + str(number) in build_names: - number += 1 - - # update identifier - identifier += str(number) - - # update the build names - build_names.add(identifier) - - return identifier - - -def simplify_subscript_input(coords, subscript_dict, return_full, merge_subs): - """ - Parameters - ---------- - coords: dict - Coordinates to write in the model file. - - subscript_dict: dict - The subscript dictionary of the model file. - - return_full: bool - If True the when coords == subscript_dict, '_subscript_dict' - will be returned - - merge_subs: list of strings - List of the final subscript range of the python array after - merging with other objects - - Returns - ------- - coords: str - The equations to generate the coord dicttionary in the model file. - - """ - - if coords == subscript_dict and return_full: - # variable defined with all the subscripts - return "_subscript_dict" - - coordsp = [] - for ndim, (dim, coord) in zip(merge_subs, coords.items()): - # find dimensions can be retrieved from _subscript_dict - if coord == subscript_dict[dim]: - # use _subscript_dict - coordsp.append(f"'{ndim}': _subscript_dict['{dim}']") - else: - # write whole dict - coordsp.append(f"'{ndim}': {coord}") - - return "{" + ", ".join(coordsp) + "}" - - -def add_entries_underscore(*dictionaries): - """ - Expands dictionaries adding new keys underscoring the white spaces - in the old ones. As the dictionaries are mutable objects this functions - will add the new entries to the already existing dictionaries with - no need to return a new one. - - Parameters - ---------- - *dictionaries: dict(s) - The dictionary or dictionaries to add the entries with underscore. - - Return - ------ - None - - """ - for dictionary in dictionaries: - keys = list(dictionary) - for name in keys: - dictionary[re.sub(" ", "_", name)] = dictionary[name] - return - - -def clean_file_names(*args): - """ - Removes special characters and makes clean file names - - Parameters - ---------- - *args: tuple - Any number of strings to to clean - - Returns - ------- - clean: list - List containing the clean strings - """ - clean = [] - for name in args: - clean.append(re.sub( - r"[\W]+", "", name.replace(" ", "_") - ).lstrip("0123456789") - ) - return clean - - -def merge_nested_dicts(original_dict, dict_to_merge): - """ - Merge dictionaries recursively, preserving common keys. - - Parameters - ---------- - original_dict: dict - Dictionary onto which the merge is executed. - - dict_to_merge: dict - Dictionary to be merged to the original_dict. - - Returns - ------- - None - """ - - for k, v in dict_to_merge.items(): - if (k in original_dict and isinstance(original_dict[k], dict) - and isinstance(dict_to_merge[k], Mapping)): - merge_nested_dicts(original_dict[k], dict_to_merge[k]) - else: - original_dict[k] = dict_to_merge[k] - - -def update_dependency(dependency, deps_dict): - """ - Update dependency in dependencies dict. - - Parameters - ---------- - dependency: str - The dependency to add to the dependency dict. - - deps_dict: dict - The dictionary of dependencies. If dependency is in deps_dict add 1 - to its value. Otherwise, add dependency to deps_dict with value 1. - - Returns - ------- - None - - """ - if dependency in deps_dict: - deps_dict[dependency] += 1 - else: - deps_dict[dependency] = 1 diff --git a/pysd/translation/vensim/vensim2py.py b/pysd/translation/vensim/vensim2py.py deleted file mode 100644 index 71005fbf..00000000 --- a/pysd/translation/vensim/vensim2py.py +++ /dev/null @@ -1,1973 +0,0 @@ -""" -These functions translate vensim .mdl file to pieces needed by the builder -module to write a python version of the model. Everything that requires -knowledge of vensim syntax should be here. -""" - -import pathlib -import re -import warnings -from io import open -from chardet import detect - -import numpy as np -import parsimonious -from parsimonious.exceptions import IncompleteParseError,\ - VisitationError,\ - ParseError - -from .. import builder, utils -from ...py_backend.external import ExtSubscript -from ...py_backend.utils import compute_shape - - -def get_file_sections(file_str): - """ - This is where we separate out the macros from the rest of the model file. - Working based upon documentation at: - https://www.vensim.com/documentation/index.html?macros.htm - - Macros will probably wind up in their own python modules eventually. - - Parameters - ---------- - file_str: str - File content to parse. - - Returns - ------- - entries: list of dictionaries - Each dictionary represents a different section of the model file, - either a macro, or the main body of the model file. The - dictionaries contain various elements: - - returns: list of strings - represents what is returned from a macro (for macros) or - empty for main model - - params: list of strings - represents what is passed into a macro (for macros) or - empty for main model - - name: string - the name of the macro, or 'main' for main body of model - - string: string - string representing the model section - - Examples - -------- - >>> get_file_sections(r'a~b~c| d~e~f| g~h~i|') - [{'returns': [], 'params': [], 'name': 'main', 'string': 'a~b~c| d~e~f| g~h~i|'}] - - """ - - # the leading 'r' for 'raw' in this string is important for - # handling backslashes properly - file_structure_grammar = _include_common_grammar( - r""" - file = encoding? (macro / main)+ - macro = ":MACRO:" _ name _ "(" _ (name _ ","? _)+ _ ":"? _ (name _ ","? _)* _ ")" ~r".+?(?=:END OF MACRO:)" ":END OF MACRO:" - main = !":MACRO:" ~r".+(?!:MACRO:)" - encoding = ~r"\{[^\}]*\}" - """ - ) - - parser = parsimonious.Grammar(file_structure_grammar) - tree = parser.parse(file_str) - - class FileParser(parsimonious.NodeVisitor): - def __init__(self, ast): - self.entries = [] - self.visit(ast) - - def visit_main(self, n, vc): - self.entries.append( - { - "name": "_main_", - "params": [], - "returns": [], - "string": n.text.strip(), - } - ) - - def visit_macro(self, n, vc): - name = vc[2] - params = vc[6] - returns = vc[10] - text = vc[13] - self.entries.append( - { - "name": name, - "params": [x.strip() for x in params.split(",") - ] if params else [], - "returns": [x.strip() for x in returns.split(",")] - if returns - else [], - "string": text.strip(), - } - ) - - def generic_visit(self, n, vc): - return "".join(filter(None, vc)) or n.text or "" - - return FileParser(tree).entries - - -def get_model_elements(model_str): - """ - Takes in a string representing model text and splits it into elements - - All newline characters were alreeady removed in a previous step. - - Parameters - ---------- - model_str : str - Model file content to read. - - Returns - ------- - entries : array of dictionaries - Each dictionary contains the components of a different model element, - separated into the equation, units, and docstring. - - Examples - -------- - # Basic Parsing: - >>> get_model_elements(r'a~b~c| d~e~f| g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'f', 'unit': 'e', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - - # Special characters are escaped within double-quotes: - >>> get_model_elements(r'a~b~c| d~e"~"~f| g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'f', 'unit': 'e"~"', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - >>> get_model_elements(r'a~b~c| d~e~"|"f| g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': '"|"f', 'unit': 'e', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - - # Double-quotes within escape groups are themselves escaped with - # backslashes: - >>> get_model_elements(r'a~b~c| d~e"\\\"~"~f| g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'f', 'unit': 'e"\\\\"~"', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - >>> get_model_elements(r'a~b~c| d~e~"\\\"|"f| g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': '"\\\\"|"f', 'unit': 'e', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - >>> get_model_elements(r'a~b~c| d~e"x\\nx"~f| g~h~|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'f', 'unit': 'e"x\\\\nx"', 'eqn': 'd'}, {'doc': '', 'unit': 'h', 'eqn': 'g'}] - - # Todo: Handle model-level or section-level documentation - >>> get_model_elements(r'*** .model doc ***~ Docstring!| d~e~f| g~h~i|') - [{'doc': 'Docstring!', 'unit': '', 'eqn': ''}, {'doc': 'f', 'unit': 'e', 'eqn': 'd'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - - # Handle control sections, returning appropriate docstring pieces - >>> get_model_elements(r'a~b~c| ****.Control***~ Simulation Control Parameters | g~h~i|') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'i', 'unit': 'h', 'eqn': 'g'}] - - # Handle the model display elements (ignore them) - >>> get_model_elements(r'a~b~c| d~e~f| \\\---///junk|junk~junk') - [{'doc': 'c', 'unit': 'b', 'eqn': 'a'}, {'doc': 'f', 'unit': 'e', 'eqn': 'd'}] - - - Notes - ----- - - Tildes and pipes are not allowed in element docstrings, but we should - still handle them there - - """ - - model_structure_grammar = _include_common_grammar( - r""" - model = (entry / section)+ sketch? - entry = element "~" element "~" doc ("~" element)? "|" - section = element "~" element "|" - sketch = ~r".*" #anything - - # Either an escape group, or a character that is not tilde or pipe - element = ( escape_group / ~r"[^~|]")* - # Anything other that is not a tilde or pipe - doc = (~r"[^~|]")* - """ - ) - - parser = parsimonious.Grammar(model_structure_grammar) - tree = parser.parse(model_str) - - class ModelParser(parsimonious.NodeVisitor): - def __init__(self, ast): - self.entries = [] - self.visit(ast) - - def visit_entry(self, n, vc): - units, lims = parse_units(vc[2].strip()) - self.entries.append( - { - "eqn": vc[0].strip(), - "unit": units, - "lims": str(lims), - "doc": vc[4].strip(), - "kind": "entry", - } - ) - - def visit_section(self, n, vc): - if vc[2].strip() != "Simulation Control Parameters": - self.entries.append( - { - "eqn": "", - "unit": "", - "lims": "", - "doc": vc[2].strip(), - "kind": "section", - } - ) - - def generic_visit(self, n, vc): - return "".join(filter(None, vc)) or n.text or "" - - return ModelParser(tree).entries - - -def _include_common_grammar(source_grammar): - common_grammar = r""" - name = basic_id / escape_group - - # This takes care of models with Unicode variable names - basic_id = id_start id_continue* - - id_start = ~r"[\w]"IU - id_continue = id_start / ~r"[0-9\'\$\s\_]" - - # between quotes, either escaped quote or character that is not a quote - escape_group = "\"" ( "\\\"" / ~r"[^\"]" )* "\"" - - _ = ~r"[\s\\]*" # whitespace character - """ - - return r""" - {source_grammar} - - {common_grammar} - """.format( - source_grammar=source_grammar, common_grammar=common_grammar - ) - - -def get_equation_components(equation_str, root_path=None): - """ - Breaks down a string representing only the equation part of a model - element. Recognizes the various types of model elements that may exist, - and identifies them. - - Parameters - ---------- - equation_str : basestring - the first section in each model element - the full equation. - - root_path: basestring - the root path of the vensim file (necessary to resolve external - data file paths) - - Returns - ------- - Returns a dictionary containing the following: - - real_name: basestring - The name of the element as given in the original vensim file - - subs: list of strings - list of subscripts or subscript elements - - expr: basestring - - kind: basestring - What type of equation have we found? - - *component* - normal model expression or constant - - *lookup* - a lookup table - - *subdef* - a subscript definition - - *data* - a data variable - - keyword: basestring or None - - Examples - -------- - >>> get_equation_components(r'constant = 25') - {'expr': '25', 'kind': 'component', 'subs': [], 'real_name': 'constant'} - - Notes - ----- - in this function we don't create python identifiers, we use real names. - This is so that when everything comes back together, we can manage - any potential namespace conflicts properly - """ - - imp_subs_func_list = [ - "get xls subscript", - "get direct subscript", - "get_xls_subscript", - "get_direct_subscript", - ] - - component_structure_grammar = _include_common_grammar( - r""" - entry = component / ext_data_definition / data_definition / test_definition / subscript_definition / lookup_definition / subscript_copy - component = name _ subscriptlist? _ "=" "="? _ expression - subscript_definition = name _ ":" _ (imported_subscript / literal_subscript / numeric_range) _ subscript_mapping_list? - ext_data_definition = name _ subscriptlist? _ keyword? _ ":=" _ expression - data_definition = name _ subscriptlist? _ keyword - lookup_definition = name _ subscriptlist? &"(" _ expression # uses - # lookahead assertion to capture whole group - test_definition = name _ subscriptlist? _ &keyword _ expression - subscript_copy = name _ "<->" _ name_mapping - - name = basic_id / escape_group - - literal_subscript = index_list - imported_subscript = imp_subs_func _ "(" _ (string _ ","? _)* ")" - numeric_range = _ (range / value) _ ("," _ (range / value) _)* - value = _ sequence_id _ - range = "(" _ sequence_id _ "-" _ sequence_id _ ")" - subscriptlist = '[' _ index_list _ ']' - subscript_mapping_list = "->" _ subscript_mapping _ ("," _ subscript_mapping _)* - subscript_mapping = (_ name_mapping _) / (_ "(" _ name_mapping _ ":" _ index_list _")" ) - - expression = ~r".*" # expression could be anything, at this point. - keyword = ":" _ basic_id _ ":" - index_list = subscript _ ("," _ subscript _)* - name_mapping = basic_id / escape_group - sequence_id = _ basic_id _ - subscript = basic_id / escape_group - imp_subs_func = ~r"(%(imp_subs)s)"IU - string = "\'" ( "\\\'" / ~r"[^\']"IU )* "\'" - """ - % {"imp_subs": "|".join(imp_subs_func_list)} - ) - - # replace any amount of whitespace with a single space - equation_str = equation_str.replace("\\t", " ") - equation_str = re.sub(r"\s+", " ", equation_str) - - parser = parsimonious.Grammar(component_structure_grammar) - - class ComponentParser(parsimonious.NodeVisitor): - def __init__(self, ast): - self.subscripts = [] - self.subscripts_compatibility = {} - self.real_name = None - self.expression = None - self.kind = None - self.keyword = None - self.visit(ast) - - def visit_subscript_definition(self, n, vc): - self.kind = "subdef" - - def visit_lookup_definition(self, n, vc): - self.kind = "lookup" - - def visit_component(self, n, vc): - self.kind = "component" - - def visit_ext_data_definition(self, n, vc): - self.kind = "component" - - def visit_data_definition(self, n, vc): - self.kind = "data" - - def visit_test_definition(self, n, vc): - # TODO: add test for test - self.kind = "test" - - def visit_keyword(self, n, vc): - self.keyword = n.text.strip() - - def visit_imported_subscript(self, n, vc): - # TODO: make this less fragile - # TODO: allow reading the subscripts from Excel - # once the model has been translated - args = [x.strip().strip("'") for x in vc[4].split(",")] - self.subscripts += ExtSubscript(*args, root=root_path).subscript - - def visit_subscript_copy(self, n, vc): - self.kind = "subdef" - subs_copy1 = vc[4].strip() - subs_copy2 = vc[0].strip() - - if subs_copy1 not in self.subscripts_compatibility: - self.subscripts_compatibility[subs_copy1] = [] - - if subs_copy2 not in self.subscripts_compatibility: - self.subscripts_compatibility[subs_copy2] = [] - - self.subscripts_compatibility[subs_copy1].append(subs_copy2) - self.subscripts_compatibility[subs_copy2].append(subs_copy1) - - def visit_subscript_mapping(self, n, vc): - - warnings.warn( - "\n Subscript mapping detected." - + "This feature works only in some simple cases." - ) - - if ":" in str(vc): - # TODO: add test for this condition - # Obtain subscript name and split by : and ( - name_mapped = str(vc).split(":")[0].split("(")[1] - else: - (name_mapped,) = vc - - if self.real_name not in self.subscripts_compatibility: - self.subscripts_compatibility[self.real_name] = [] - self.subscripts_compatibility[self.real_name].append( - name_mapped.strip()) - - def visit_range(self, n, vc): - subs_start = vc[2].strip() - subs_end = vc[6].strip() - - # get the common prefix and the starting and - # ending number of the numeric range - subs_start = re.findall(r"\d+|\D+", subs_start) - subs_end = re.findall(r"\d+|\D+", subs_end) - prefix_start = "".join(subs_start[:-1]) - prefix_end = "".join(subs_end[:-1]) - num_start = int(subs_start[-1]) - num_end = int(subs_end[-1]) - - if not prefix_start or not prefix_end: - raise ValueError( - "\nA numeric range must contain at least one letter.") - elif num_start >= num_end: - raise ValueError( - "\nThe number of the first subscript value must be " - "lower than the second subscript value in a " - "subscript numeric range.") - elif (prefix_start != prefix_end - or subs_start[0].isdigit() - or subs_end[0].isdigit()): - raise ValueError( - "\nOnly matching names ending in numbers are valid.") - - for i in range(num_start, num_end + 1): - s = prefix_start + str(i) - self.subscripts.append(s.strip()) - - def visit_value(self, n, vc): - self.subscripts.append(vc[1].strip()) - - def visit_name(self, n, vc): - (name,) = vc - self.real_name = name.strip() - return self.real_name - - def visit_subscript(self, n, vc): - (subscript,) = vc - self.subscripts.append(subscript.strip()) - return subscript.strip() - - def visit_expression(self, n, vc): - self.expression = n.text.strip() - - def generic_visit(self, n, vc): - return "".join(filter(None, vc)) or n.text - - def visit__(self, n, vc): - return " " - - try: - tree = parser.parse(equation_str) - parse_object = ComponentParser(tree) - except (IncompleteParseError, VisitationError, ParseError) as err: - # this way we get the element name and equation and is easier - # to detect the error in the model file - raise ValueError( - err.args[0] + "\n\n" - "\nError when parsing definition:\n\t %s\n\n" - "probably used definition is invalid or not integrated..." - "\nSee parsimonious output above." % (equation_str) - ) - - return { - "real_name": parse_object.real_name, - "subs": parse_object.subscripts, - "subs_compatibility": parse_object.subscripts_compatibility, - "expr": parse_object.expression, - "kind": parse_object.kind, - "keyword": parse_object.keyword, - } - - -def parse_sketch_line(sketch_line, namespace): - """ - This syntax parses a single line of the Vensim sketch at a time. - - Not all possibilities can be tested, so this gammar may be considered - experimental for now - - """ - - sketch_grammar = _include_common_grammar( - r""" - line = var_definition / view_intro / view_title / view_definition / arrow / flow / other_objects / anything - view_intro = ~r"\s*Sketch.*?names$" / ~r"^V300.*?ignored$" - view_title = "*" view_name - view_name = ~r"(?<=\*)[^\n]+$" - view_definition = "$" color "," digit "," font_properties "|" ( ( color / ones_and_dashes ) "|")* view_code - var_definition = var_code "," var_number "," var_name "," position "," var_box_type "," arrows_in_allowed "," hide_level "," var_face "," var_word_position "," var_thickness "," var_rest_conf ","? ( ( ones_and_dashes / color) ",")* font_properties? ","? extra_bytes? - # elements used in a line defining the properties of a variable or stock - var_name = element - var_name = ~r"(?<=,)[^,]+(?=,)" - var_number = digit - var_box_type = ~r"(?<=,)\d+,\d+,\d+(?=,)" # improve this regex - arrows_in_allowed = ~r"(?<=,)\d+(?=,)" # if this is an even number, - # it's a shadow variable - hide_level = digit - var_face = digit - var_word_position = ~r"(?<=,)\-*\d+(?=,)" - var_thickness = digit - var_rest_conf = digit "," ~r"\d+" - extra_bytes = ~r"\d+,\d+,\d+,\d+,\d+,\d+" # required since Vensim 8.2.1 - arrow = arrow_code "," digit "," origin_var "," destination_var "," (digit ",")+ (ones_and_dashes ",")? ((color ",") / ("," ~r"\d+") / (font_properties "," ~r"\d+"))* "|(" position ")|" - # arrow origin and destination (this may be useful if further - # parsing is required) - origin_var = digit - destination_var = digit - # flow arrows - flow = source_or_sink_or_plot / flow_arrow - # if you want to extend the parsing, these three would be a good - # starting point (they are followed by "anything") - source_or_sink_or_plot = multipurpose_code "," anything - flow_arrow = flow_arrow_code "," anything - other_objects = other_objects_code "," anything - # fonts - font_properties = font_name? "|" font_size "|" font_style? "|" color - font_style = "B" / "I" / "U" / "S" / "V" # italics, bold, underline, etc - font_size = ~r"\d+" # this needs to be made a regex to match any font - font_name = ~r"(?<=,)[^\|\d]+(?=\|)" - # x and y within the view layout. This may be useful if further - # parsing is required - position = ~r"-*\d+,-*\d+" - # rgb color (e.g. 255-255-255) - color = ~r"((?>> parse_units('Widgets/Month [-10,10,1]') - ('Widgets/Month', (-10,10,1)) - - >>> parse_units('Month [0,?]') - ('Month', [-10, None]) - - >>> parse_units('Widgets [0,100]') - ('Widgets', (0, 100)) - - >>> parse_units('Widgets') - ('Widgets', (None, None)) - - >>> parse_units('[0, 100]') - ('', (0, 100)) - - """ - if not len(units_str): - return units_str, (None, None) - - if units_str[-1] == "]": - units, lims = units_str.rsplit("[") # types: str, str - else: - units = units_str - lims = "?, ?]" - - lims = tuple( - [float(x) if x.strip() != "?" else None for x in lims.strip("]").split( - ",")] - ) - - return units.strip(), lims - - -functions = { - # element-wise functions - "abs": {"name": "np.abs", "module": "numpy"}, - "min": {"name": "np.minimum", "module": "numpy"}, - "max": {"name": "np.maximum", "module": "numpy"}, - "exp": {"name": "np.exp", "module": "numpy"}, - "sin": {"name": "np.sin", "module": "numpy"}, - "cos": {"name": "np.cos", "module": "numpy"}, - "tan": {"name": "np.tan", "module": "numpy"}, - "arcsin": {"name": "np.arcsin", "module": "numpy"}, - "arccos": {"name": "np.arccos", "module": "numpy"}, - "arctan": {"name": "np.arctan", "module": "numpy"}, - "sinh": {"name": "np.sinh", "module": "numpy"}, - "cosh": {"name": "np.cosh", "module": "numpy"}, - "tanh": {"name": "np.tanh", "module": "numpy"}, - "sqrt": {"name": "np.sqrt", "module": "numpy"}, - "integer": {"name": "integer", "module": "functions"}, - "quantum": {"name": "quantum", "module": "functions"}, - "modulo": {"name": "modulo", "module": "functions"}, - "xidz": {"name": "xidz", "module": "functions"}, - "zidz": {"name": "zidz", "module": "functions"}, - "ln": {"name": "np.log", "module": "numpy"}, - "log": {"name": "log", "module": "functions"}, - "lognormal": {"name": "np.random.lognormal", "module": "numpy"}, - "random normal": {"name": "bounded_normal", "module": "functions"}, - "poisson": {"name": "np.random.poisson", "module": "numpy"}, - "exprnd": {"name": "np.random.exponential", "module": "numpy"}, - "random 0 1": { - "name": "np.random.uniform", - "parameters": [ - {"name": "0", "type": "predef"}, - {"name": "1", "type": "predef"} - ], - "module": "numpy"}, - "random uniform": { - "name": "np.random.uniform", - "parameters": [ - {"name": "m"}, - {"name": "x"}, - {"name": "s", "type": "ignore"} - ], - "module": "numpy"}, - "elmcount": { - "name": "len", - "parameters": [ - {"name": "subs_range", "type": "subs_range_to_list"}, - ] - }, - "if then else": { - "name": "if_then_else", - "parameters": [ - {"name": "condition"}, - {"name": "val_if_true", "type": "lambda"}, - {"name": "val_if_false", "type": "lambda"}, - ], - "module": "functions", - }, - "step": { - "name": "step", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "value"}, - {"name": "tstep"}, - ], - "module": "functions", - }, - "pulse": { - "name": "pulse", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "start"}, - {"name": "duration"}, - ], - "module": "functions", - }, - # time, start, duration, repeat_time, end - "pulse train": { - "name": "pulse_train", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "start"}, - {"name": "duration"}, - {"name": "repeat_time"}, - {"name": "end"}, - ], - "module": "functions", - }, - "ramp": { - "name": "ramp", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "slope"}, - {"name": "start"}, - {"name": "finish"}, - ], - "module": "functions", - }, - "active initial": { - "name": "active_initial", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "expr", "type": "lambda"}, - {"name": "init_val"}, - ], - "module": "functions", - }, - "game": "", # In the future, may have an actual `functions.game` to pass - # vector functions - "sum": {"name": "sum", "module": "functions"}, - "prod": {"name": "prod", "module": "functions"}, - "vmin": {"name": "vmin", "module": "functions"}, - "vmax": {"name": "vmax", "module": "functions"}, - # matricial functions - "invert matrix": { - "name": "invert_matrix", - "parameters": [ - {"name": "mat"}, - {"name": "n", "type": "ignore"} - # we can safely ignore VENSIM's n parameter - ], - "module": "functions"}, - # TODO functions/stateful objects to be added - "get time value": { - "name": "not_implemented_function", - "module": "functions", - "original_name": "GET TIME VALUE", - }, - # https://github.com/JamesPHoughton/pysd/issues/263 - "allocate by priority": { - "name": "not_implemented_function", - "module": "functions", - "original_name": "ALLOCATE BY PRIORITY", - }, - # https://github.com/JamesPHoughton/pysd/issues/266 - "vector select": { - "name": "not_implemented_function", - "module": "functions", - "original_name": "VECTOR SELECT", - }, - # https://github.com/JamesPHoughton/pysd/issues/265 - "shift if true": { - "name": "not_implemented_function", - "module": "functions", - "original_name": "SHIFT IF TRUE", - }, -} - - -# list of fuctions that accept a dimension to apply over -vectorial_funcs = ["sum", "prod", "vmax", "vmin"] - -# other functions -functions_utils = { - "lookup": {"name": "lookup", "module": "functions"}, - "rearrange": {"name": "rearrange", "module": "utils"}, - "DataArray": {"name": "xr.DataArray", "module": "xarray"}, -} - -# logical operators (bool? operator bool) -in_logical_ops = { - ":and:": { - "name": "logical_and", - "module": "functions" - }, - ":or:": { - "name": "logical_or", - "module": "functions" - } -} - -pre_logical_ops = { - ":not:": { - "name": "np.logical_not", - "module": "numpy" - } -} - -data_ops = { - "get data at time": "", - "get data between times": "", - "get data last time": "", - "get data max": "", - "get data min": "", - "get data median": "", - "get data mean": "", - "get data stdv": "", - "get data total points": "", -} - -builders = { - "integ": lambda element, subscript_dict, args: - builder.add_stock( - identifier=element["py_name"], - expression=args[0], - initial_condition=args[1], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "delay1": lambda element, subscript_dict, args: - builder.add_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[0], - order="1", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "delay1i": lambda element, subscript_dict, args: - builder.add_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2], - order="1", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "delay3": lambda element, subscript_dict, args: - builder.add_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[0], - order="3", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "delay3i": lambda element, subscript_dict, args: - builder.add_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2], - order="3", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "delay fixed": lambda element, subscript_dict, args: - builder.add_delay_f( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2], - deps=element["dependencies"] - ), - "delay n": lambda element, subscript_dict, args: - builder.add_n_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2], - order=args[3], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "forecast": lambda element, subscript_dict, args: - builder.add_forecast( - identifier=element["py_name"], - forecast_input=args[0], - average_time=args[1], - horizon=args[2], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "sample if true": lambda element, subscript_dict, args: - builder.add_sample_if_true( - identifier=element["py_name"], - condition=args[0], - actual_value=args[1], - initial_value=args[2], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "smooth": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[0], - order="1", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "smoothi": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2], - order="1", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "smooth3": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[0], - order="3", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "smooth3i": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2], - order="3", - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "smooth n": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2], - order=args[3], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "trend": lambda element, subscript_dict, args: - builder.add_n_trend( - identifier=element["py_name"], - trend_input=args[0], - average_time=args[1], - initial_trend=args[2], - subs=element["subs"], - merge_subs=element["merge_subs"], - deps=element["dependencies"] - ), - "get xls data": lambda element, subscript_dict, args: - builder.add_ext_data( - identifier=element["py_name"], - file_name=args[0], - tab=args[1], - time_row_or_col=args[2], - cell=args[3], - subs=element["subs"], - subscript_dict=subscript_dict, - merge_subs=element["merge_subs"], - keyword=element["keyword"], - ), - "get xls constants": lambda element, subscript_dict, args: - builder.add_ext_constant( - identifier=element["py_name"], - file_name=args[0], - tab=args[1], - cell=args[2], - subs=element["subs"], - subscript_dict=subscript_dict, - merge_subs=element["merge_subs"], - ), - "get xls lookups": lambda element, subscript_dict, args: - builder.add_ext_lookup( - identifier=element["py_name"], - file_name=args[0], - tab=args[1], - x_row_or_col=args[2], - cell=args[3], - subs=element["subs"], - subscript_dict=subscript_dict, - merge_subs=element["merge_subs"], - ), - "initial": lambda element, subscript_dict, args: - builder.add_initial( - identifier=element["py_name"], - value=args[0], - deps=element["dependencies"] - ), - "a function of": lambda element, subscript_dict, args: - builder.add_incomplete( - element["real_name"], args - ), -} - -# direct and xls methods are identically implemented in PySD -builders["get direct data"] = builders["get xls data"] -builders["get direct lookups"] = builders["get xls lookups"] -builders["get direct constants"] = builders["get xls constants"] - -# expand dictionaries to detect _ in Vensim def -utils.add_entries_underscore(functions, data_ops, builders) - - -def parse_general_expression(element, namespace={}, subscript_dict={}, - macro_list=None, elements_subs_dict={}, - subs_compatibility={}): - """ - Parses a normal expression - # its annoying that we have to construct and compile the grammar every - # time... - - Parameters - ---------- - element: dictionary - - namespace : dictionary - - subscript_dict : dictionary - - macro_list: list of dictionaries - [{'name': 'M', 'py_name':'m', 'filename':'path/to/file', 'args': - ['arg1', 'arg2']}] - - elements_subs_dict : dictionary - The dictionary with element python names as keys and their merged - subscripts as values. - - subs_compatibility : dictionary - The dictionary storing the mapped subscripts - - Returns - ------- - translation - - new_elements: list of dictionaries - If the expression contains builder functions, those builders will - create new elements to add to our running list (that will eventually - be output to a file) such as stock initialization and derivative - funcs, etc. - - - Examples - -------- - >>> parse_general_expression({'expr': 'INTEG (FlowA, -10)', - ... 'py_name':'test_stock', - ... 'subs':None}, - ... {'FlowA': 'flowa'}), - ({'kind': 'component', 'py_expr': "_state['test_stock']"}, - [{'kind': 'implicit', - 'subs': None, - 'doc': 'Provides initial conditions for test_stock function', - 'py_name': 'init_test_stock', - 'real_name': None, - 'unit': 'See docs for test_stock', - 'py_expr': '-10'}, - {'py_name': 'dtest_stock_dt', - 'kind': 'implicit', - 'py_expr': 'flowa', - 'real_name': None}]) - - """ - - element["dependencies"] = dict() - # spaces important for word-based operators - in_ops = { - "+": "+", "-": "-", "*": "*", "/": "/", "^": "**", "=": "==", - "<=": "<=", "<>": "!=", "<": "<", ">=": ">=", ">": ">"} - - pre_ops = { - "-": "-", - "+": " " # space is important, so that and empty string doesn't - # slip through generic - } - - pre_ops = {"-": "-", "+": " ", ":not:": " not "} - - # in the following, if lists are empty use non-printable character - # everything needs to be escaped before going into the grammar, - # in case it includes quotes - sub_names_list = [re.escape(x) for x in subscript_dict.keys()] or ["\\a"] - sub_elems_list = [ - re.escape(y).replace('"', "") for x in subscript_dict.values() for y - in x] or ["\\a"] - in_ops_list = [re.escape(x) for x in in_ops.keys()] - pre_ops_list = [re.escape(x) for x in pre_ops.keys()] - if macro_list is not None and len(macro_list) > 0: - macro_names_list = [re.escape(x["name"]) for x in macro_list] - else: - macro_names_list = ["\\a"] - - expression_grammar = _include_common_grammar( - r""" - expr_type = array / expr / empty - expr = _ pre_oper? _ (lookup_with_def / build_call / macro_call / call / lookup_call / parens / number / string / reference / nan) _ (in_oper _ expr)? - subs_expr = subs _ in_oper _ subs - - logical_expr = logical_in_expr / logical_pre_expr / logical_parens / subs_expr - logical_in_expr = (logical_pre_expr / logical_parens / subs_expr / expr) (_ in_logical_oper _ (logical_pre_expr / logical_parens / subs_expr / expr))+ - logical_pre_expr = pre_logical_oper _ (logical_parens / subs_expr / expr) - - lookup_with_def = ~r"(WITH\ LOOKUP)"I _ "(" _ expr _ "," _ "(" _ ("[" ~r"[^\]]*" "]" _ ",")? ( "(" _ expr _ "," _ expr _ ")" _ ","? _ )+ _ ")" _ ")" - - lookup_call = lookup_call_subs _ parens - lookup_call_subs = (id _ subscript_list) / id # check first for subscript - - nan = ":NA:" - number = ("+"/"-")? ~r"\d+\.?\d*([eE][+-]?\d+)?" - range = _ "[" ~r"[^\]]*" "]" _ "," - - arguments = ((logical_expr / (subs_range !(_ id)) / expr) _ ","? _)* - parens = "(" _ expr _ ")" - logical_parens = "(" _ logical_expr _ ")" - - call = func _ "(" _ arguments _ ")" - build_call = builder _ "(" _ arguments _ ")" - macro_call = macro _ "(" _ arguments _ ")" - - reference = (id _ subscript_list) / id # check first for subscript - subscript_list = "[" _ ~"\""? _ (subs _ ~"\""? _ "!"? _ ","? _)+ _ "]" - - array = (number _ ("," / ";")? _ "\\"? _)+ !~r"." # negative lookahead for - # anything other than an array - string = "\'" ( "\\\'" / ~r"[^\']"IU )* "\'" - - id = ( basic_id / escape_group ) - - subs = ~r"(%(subs)s)"IU # subscript names and elements (if none, use - # non-printable character) - subs_range = ~r"(%(subs_range)s)"IU # subscript names - func = ~r"(%(funcs)s)"IU # functions (case insensitive) - in_oper = ~r"(%(in_ops)s)"IU # infix operators (case insensitive) - pre_oper = ~r"(%(pre_ops)s)"IU # prefix operators (case insensitive) - in_logical_oper = ~r"(%(in_logical_ops)s)"IU # infix operators (case - # insensitive) - pre_logical_oper = ~r"(%(pre_logical_ops)s)"IU # prefix operators (case - # insensitive) - builder = ~r"(%(builders)s)"IU # builder functions (case insensitive) - macro = ~r"(%(macros)s)"IU # macros from model file (if none, use - # non-printable character) - - empty = "" # empty string - """ % { - # In the following, we have to sort keywords in decreasing order - # of length so that the peg parser doesn't quit early when - # finding a partial keyword - 'subs': '|'.join(reversed(sorted(sub_names_list + sub_elems_list, - key=len))), - 'subs_range': '|'.join(reversed(sorted(sub_names_list, key=len))), - 'funcs': '|'.join(reversed(sorted(functions.keys(), key=len))), - 'in_ops': '|'.join(reversed(sorted(in_ops_list, key=len))), - 'pre_ops': '|'.join(reversed(sorted(pre_ops_list, key=len))), - 'in_logical_ops': '|'.join(reversed(sorted(in_logical_ops.keys(), - key=len))), - 'pre_logical_ops': '|'.join(reversed(sorted(pre_logical_ops.keys(), - key=len))), - 'builders': '|'.join(reversed(sorted(builders.keys(), key=len))), - 'macros': '|'.join(reversed(sorted(macro_names_list, key=len))) - }) - - parser = parsimonious.Grammar(expression_grammar) - - class ExpressionParser(parsimonious.NodeVisitor): - # TODO: at some point, we could make the 'kind' identification - # recursive on expression, so that if an expression is passed into - # a builder function, the information about whether it is a constant, - # or calls another function, goes with it. - - def __init__(self, ast): - self.translation = "" - self.subs = None # the subscript list if given - self.lookup_subs = [] - self.apply_dim = set() # the dimensions with ! if given - self.kind = "constant" # change if we reference anything else - self.new_structure = [] - self.append = "" - self.lookup_append = [] - self.arguments = None - self.in_oper = None - self.args = [] - self.logical_op = None - self.to_float = False # convert subseted reference to float - self.visit(ast) - - def visit_expr_type(self, n, vc): - s = "".join(filter(None, vc)).strip() - self.translation = s - - def visit_expr(self, n, vc): - s = "".join(filter(None, vc)).strip() - self.translation = s - return s - - def visit_call(self, n, vc): - self.kind = "component" - - function_name = vc[0].lower() - arguments = vc[4] - - # add dimensions as last argument - if self.apply_dim and function_name in vectorial_funcs: - arguments += ["dim=" + str(tuple(self.apply_dim))] - self.apply_dim = set() - - if re.match(r"active(_|\s)initial", function_name): - ghost_name, new_structure = builder.build_active_initial_deps( - element["py_name"], arguments, element["dependencies"]) - element["dependencies"] = {ghost_name: 1} - self.new_structure += new_structure - - return builder.build_function_call( - functions[function_name], - arguments, element["dependencies"]) - - def visit_in_oper(self, n, vc): - return in_ops[n.text.lower()] - - def visit_pre_oper(self, n, vc): - return pre_ops[n.text.lower()] - - def visit_logical_in_expr(self, n, vc): - # build logical in expression (or, and) - expr = "".join(vc) - expr_low = expr.lower() - - if ":and:" in expr_low and ":or:" in expr_low: - raise ValueError( - "\nError when parsing %s with equation\n\t %s\n\n" - "mixed definition of logical operators :OR: and :AND:" - "\n Use parethesis to avoid confusions." % ( - element['real_name'], element['eqn']) - ) - elif ":and:" in expr_low: - expr = re.split(":and:", expr, flags=re.IGNORECASE) - op = ':and:' - elif ":or:" in expr_low: - expr = re.split(":or:", expr, flags=re.IGNORECASE) - op = ':or:' - - return builder.build_function_call(in_logical_ops[op], expr) - - def visit_logical_pre_expr(self, n, vc): - # build logical pre expression (not) - return builder.build_function_call(pre_logical_ops[vc[0].lower()], - [vc[-1]]) - - def visit_logical_parens(self, n, vc): - # we can forget about the parenthesis in logical expressions - # as we pass them as arguments to other functions: - # (A or B) and C -> logical_and(logical_or(A, B), C) - return vc[2] - - def visit_reference(self, n, vc): - self.kind = "component" - - py_expr = vc[0] + "()" + self.append - self.append = "" - - if self.to_float: - # convert element to float after subscript subsetting - self.to_float = False - return "float(" + py_expr.replace(".reset_coords(drop=True", - "") - elif self.subs: - if elements_subs_dict[vc[0]] != self.subs: - py_expr = builder.build_function_call( - functions_utils["rearrange"], - [py_expr, repr(self.subs), "_subscript_dict"], - ) - - mapping = self.subs.copy() - for i, sub in enumerate(self.subs): - if sub in subs_compatibility: - for compatible in subs_compatibility[sub]: - if compatible in element["subs"]: - mapping[i] = compatible - - if self.subs != mapping: - py_expr = builder.build_function_call( - functions_utils["rearrange"], - [py_expr, repr(mapping), "_subscript_dict"], - ) - - self.subs = None - - return py_expr - - def visit_lookup_call_subs(self, n, vc): - # necessary if a lookup dimension is subselected but we have - # other reference objects as arguments - self.lookup_append.append(self.append) - self.to_float = False # argument may have dims, cannot convert - self.append = "" - - # recover subs for lookup to avoid using them for arguments - if self.subs: - self.lookup_subs.append(self.subs) - self.subs = None - else: - self.lookup_subs.append(None) - - return vc[0] - - def visit_lookup_call(self, n, vc): - lookup_append = self.lookup_append.pop() - lookup_subs = self.lookup_subs.pop() - py_expr = "".join([x.strip(",") for x in vc]) + lookup_append - - if lookup_subs and elements_subs_dict[vc[0]] != lookup_subs: - dims = [ - utils.find_subscript_name(subscript_dict, sub) - for sub in lookup_subs - ] - return builder.build_function_call( - functions_utils["rearrange"], - [py_expr, repr(dims), "_subscript_dict"], - ) - - return py_expr - - def visit_id(self, n, vc): - subelement = namespace[n.text.strip()] - utils.update_dependency(subelement, element["dependencies"]) - return subelement - - def visit_lookup_with_def(self, n, vc): - """This exists because vensim has multiple ways of doing lookups. - Which is frustrating.""" - x_val = vc[4] - pairs = vc[11] - mixed_list = pairs.replace("(", "").replace(")", "").split(",") - xs = mixed_list[::2] - ys = mixed_list[1::2] - arguments = [x_val, "[" + ",".join(xs) + "]", "[" + ",".join(ys) + - "]"] - return builder.build_function_call(functions_utils["lookup"], - arguments) - - def visit_array(self, n, vc): - # first test handles when subs is not defined - if "subs" in element and element["subs"]: - coords = utils.make_coord_dict( - element["subs"], subscript_dict, terse=False - ) - if ";" in n.text or "," in n.text: - text = n.text.strip(";").replace(" ", "").replace( - ";", ",").replace("\\", "") - data = np.array([float(s) for s in text.split(",")]) - data = data.reshape(compute_shape(coords)) - datastr = ( - np.array2string(data, separator=",") - .replace("\n", "") - .replace(" ", "") - ) - else: - datastr = n.text - - return builder.build_function_call( - functions_utils["DataArray"], - [datastr, - utils.simplify_subscript_input( - coords, subscript_dict, - return_full=True, - merge_subs=element["merge_subs"]), - repr(element["merge_subs"])] - ) - else: - return n.text.replace(" ", "") - - def visit_subs_expr(self, n, vc): - # visit a logical comparation between subscripts - return builder.build_function_call( - functions_utils["DataArray"], [ - f"_subscript_dict['{vc[0]}']", - "{"+f"'{vc[0]}': _subscript_dict['{vc[0]}']"+"}", - f"'{vc[0]}'"] - ) + vc[2] + builder.build_function_call( - functions_utils["DataArray"], [ - f"_subscript_dict['{vc[4]}']", - "{"+f"'{vc[4]}': _subscript_dict['{vc[4]}']"+"}", - f"'{vc[4]}'"] - ) - - def visit_subscript_list(self, n, vc): - refs = vc[4] - subs = [x.strip() for x in refs.split(",")] - coordinates = [ - sub if sub not in subscript_dict and sub[-1] != "!" else False - for sub in subs - ] - - # Implements basic "!" subscript functionality in Vensim. - # Does NOT work for matrix diagonals in - # FUNC(variable[sub1!,sub1!]) functions - self.apply_dim.update(["%s" % s.strip("!") for s in subs if s[-1] - == "!"]) - - if any(coordinates): - coords, subs2 = [], [] - for coord, sub in zip(coordinates, subs): - if coord: - # subset coord - coords.append("'%s'" % coord) - else: - # do not subset coord - coords.append(":") - subs2.append(sub.strip("!")) - - if subs2: - self.subs = subs2 - else: - # convert subseted element to float (avoid using 0D xarray) - self.to_float = True - - self.append = ".loc[%s].reset_coords(drop=True)" % (", ".join( - coords)) - - else: - self.subs = ["%s" % s.strip("!") for s in subs] - - return "" - - def visit_build_call(self, n, vc): - # use only the dict with the final subscripts - # needed for the good working of externals - subs_dict = { - k: subscript_dict[k] for k in - element["merge_subs"] - } - # add subscript ranges given in expr - subs_dict.update({ - sub: subscript_dict[sub] for sub in element['subs'] - if sub in subscript_dict - }) - - self.kind = "component" - builder_name = vc[0].strip().lower() - - name, structure = builders[builder_name]( - element, subs_dict, vc[4]) - - self.new_structure += structure - - if "lookups" in builder_name: - self.arguments = "x" - self.kind = "lookup" - element["dependencies"].update({ - "__external__": None, "__lookup__": None}) - elif "constant" in builder_name: - # External constants - self.kind = "constant" - element["dependencies"]["__external__"] = None - elif "data" in builder_name: - # External data - self.kind = "component_ext_data" - element["dependencies"]["__external__"] = None - element["dependencies"]["time"] = 1 - elif "a function of" not in builder_name: - element["dependencies"] = {structure[-1]["py_name"]: 1} - - return name - - def visit_macro_call(self, n, vc): - call = vc[0] - arglist = vc[4] - self.kind = "component" - py_name = utils.make_python_identifier(call) - macro = [x for x in macro_list if x["py_name"] == py_name][ - 0 - ] # should match once - name, structure = builder.add_macro( - element["py_name"], - macro["py_name"], macro["file_name"], - macro["params"], arglist, element["dependencies"] - ) - element["dependencies"] = {structure[-1]["py_name"]: 1} - self.new_structure += structure - return name - - def visit_arguments(self, n, vc): - arglist = [x.strip(",") for x in vc] - return arglist - - def visit__(self, n, vc): - """Handles whitespace characters""" - return "" - - def visit_nan(self, n, vc): - builder.Imports.add("numpy") - return "np.nan" - - def visit_empty(self, n, vc): - warnings.warn(f"Empty expression for '{element['real_name']}''.") - return "None" - - def generic_visit(self, n, vc): - return "".join(filter(None, vc)) or n.text - - try: - tree = parser.parse(element["expr"]) - parse_object = ExpressionParser(tree) - except (IncompleteParseError, VisitationError, ParseError) as err: - # this way we get the element name and equation and is easier - # to detect the error in the model file - raise ValueError( - err.args[0] + "\n\n" - "\nError when parsing %s with equation\n\t %s\n\n" - "probably a used function is not integrated..." - "\nSee parsimonious output above." % (element["real_name"], - element["eqn"]) - ) - - return ( - { - "py_expr": parse_object.translation, - "kind": parse_object.kind, - "arguments": parse_object.arguments or "", - }, - parse_object.new_structure, - ) - - -def parse_lookup_expression(element, subscript_dict): - """This syntax parses lookups that are defined with their own element""" - - element["dependencies"] = dict() - - lookup_grammar = r""" - lookup = _ "(" _ (regularLookup / excelLookup) _ ")" - regularLookup = range? _ ( "(" _ number _ "," _ number _ ")" _ ","? _ )+ - excelLookup = ~"GET( |_)(XLS|DIRECT)( |_)LOOKUPS"I _ "(" (args _ ","? _)+ ")" - args = ~r"[^,()]*" - number = ("+"/"-")? ~r"\d+\.?\d*(e[+-]\d+)?" - _ = ~r"[\s\\]*" #~r"[\ \t\n]*" #~r"[\s\\]*" # whitespace character - range = _ "[" ~r"[^\]]*" "]" _ "," - """ - parser = parsimonious.Grammar(lookup_grammar) - tree = parser.parse(element["expr"]) - - class LookupParser(parsimonious.NodeVisitor): - def __init__(self, ast): - self.translation = "" - self.new_structure = [] - self.visit(ast) - - def visit__(self, n, vc): - # remove whitespace - return "" - - def visit_regularLookup(self, n, vc): - - pairs = max(vc, key=len) - mixed_list = pairs.replace("(", "").replace(")", "").split(",") - xs = mixed_list[::2] - ys = mixed_list[1::2] - arguments = ["x", "[" + ",".join(xs) + "]", "[" + ",".join(ys) + - "]"] - self.translation = builder.build_function_call( - functions_utils["lookup"], arguments - ) - - def visit_excelLookup(self, n, vc): - arglist = vc[3].split(",") - arglist = [arg.replace("\\ ", "") for arg in arglist] - # use only the dict with the final subscripts - # needed for the good working of externals - subs_dict = { - k: subscript_dict[k] for k in - element["merge_subs"] - } - # add subscript ranges given in expr - subs_dict.update({ - sub: subscript_dict[sub] for sub in element['subs'] - if sub in subscript_dict - }) - trans, structure = builders["get xls lookups"]( - element, subs_dict, arglist - ) - element["dependencies"]["__external__"] = None - - self.translation = trans - self.new_structure += structure - - def generic_visit(self, n, vc): - return "".join(filter(None, vc)) or n.text - - parse_object = LookupParser(tree) - return ( - {"py_expr": parse_object.translation, "arguments": "x"}, - parse_object.new_structure, - ) - - -def translate_section(section, macro_list, sketch, root_path, subview_sep=""): - - model_elements = get_model_elements(section["string"]) - - # extract equation components - model_docstring = "" - for entry in model_elements: - if entry["kind"] == "entry": - entry.update(get_equation_components(entry["eqn"], root_path)) - elif entry["kind"] == "section": - model_docstring += entry["doc"] - - # make python identifiers and track for namespace conflicts - namespace = {"TIME": "time", "Time": "time"} # Initialize with builtins - - # add macro parameters when parsing a macro section - for param in section["params"]: - utils.make_python_identifier(param, namespace) - - # add macro functions to namespace - for macro in macro_list: - if macro["name"] != "_main_": - utils.make_python_identifier(macro["name"], namespace) - - # Create a namespace for the subscripts as these aren't used to - # create actual python functions, but are just labels on arrays, - # they don't actually need to be python-safe - # Also creates a dictionary with all the subscript that are mapped - - subscript_dict = {} - subs_compatibility_dict = {} - for e in model_elements: - if e["kind"] == "subdef": - subscript_dict[e["real_name"]] = e["subs"] - for compatible in e["subs_compatibility"]: - subs_compatibility_dict[compatible] =\ - set(e["subs_compatibility"][compatible]) - # check if copy - if not subscript_dict[compatible]: - # copy subscript to subscript_dict - subscript_dict[compatible] =\ - subscript_dict[e["subs_compatibility"][compatible][0]] - - elements_subs_dict = {} - # add model elements - for element in model_elements: - if element["kind"] not in ["subdef", "section"]: - element["py_name"] = utils.make_python_identifier( - element["real_name"], namespace) - # dictionary to save the subscripts of each element so we can avoid - # using utils.rearrange when calling them with the same dimensions - if element["py_name"] in elements_subs_dict: - elements_subs_dict[element["py_name"]].append(element["subs"]) - else: - elements_subs_dict[element["py_name"]] = [element["subs"]] - - elements_subs_dict = { - el: utils.make_merge_list(elements_subs_dict[el], subscript_dict, el) - for el in elements_subs_dict - } - - for element in model_elements: - if "py_name" in element and element["py_name"] in elements_subs_dict: - element["merge_subs"] =\ - elements_subs_dict[element["py_name"]] - else: - element["merge_subs"] = None - - # Parse components to python syntax. - for element in model_elements: - if element["kind"] == "component" and "py_expr" not in element: - # TODO: if there is new structure, - # it should be added to the namespace... - translation, new_structure = parse_general_expression( - element, - namespace=namespace, - subscript_dict=subscript_dict, - macro_list=macro_list, - subs_compatibility=subs_compatibility_dict, - elements_subs_dict=elements_subs_dict - ) - element.update(translation) - model_elements += new_structure - - elif element["kind"] == "data": - element["eqn"] = element["expr"] = element["arguments"] = "" - element["py_expr"], new_structure = builder.add_tab_data( - element["py_name"], element["real_name"], - element["subs"], subscript_dict, element["merge_subs"], - element["keyword"]) - - element["dependencies"] = {"time": 1, "__data__": None} - model_elements += new_structure - - elif element["kind"] == "lookup": - translation, new_structure = parse_lookup_expression( - element, - subscript_dict=subscript_dict - ) - element.update(translation) - model_elements += new_structure - - element["dependencies"]["__lookup__"] = None - - # send the pieces to be built - build_elements = builder.merge_partial_elements([ - e for e in model_elements if e["kind"] not in ["subdef", "test", - "section"] - ]) - - dependencies = { - element["py_name"]: element["dependencies"] - - for element in build_elements - if element["dependencies"] is not None - } - - # macros are built in their own separate files, and their inputs and - # outputs are put in views/subviews - if sketch and (section["name"] == "_main_"): - module_elements = _classify_elements_by_module(sketch, namespace, - subview_sep) - if (len(module_elements.keys()) == 1) \ - and (isinstance(module_elements[list(module_elements)[0]], list)): - warnings.warn( - "Only a single view with no subviews was detected. The model" - " will be built in a single file.") - else: - builder.build_modular_model( - build_elements, - subscript_dict, - namespace, - dependencies, - section["file_path"], - module_elements, - ) - return section["file_path"] - - builder.build(build_elements, subscript_dict, namespace, dependencies, - section["file_path"]) - - return section["file_path"] - - -def _classify_elements_by_module(sketch, namespace, subview_sep): - """ - Takes the Vensim sketch as a string, parses it (line by line) and - returns a dictionary containing the views/subviews as keys and the model - elements that belong to each view/subview inside a list as values. - - Parameters - ---------- - sketch: string - Representation of the Vensim Sketch as a string. - - namespace: dict - Translation from original model element names (keys) to python - safe function identifiers (values). - - subview_sep: list - Characters used to split view names into view + subview - (e.g. if a view is named ENERGY.Demand and suview_sep is set to ".", - then the Demand subview would be placed inside the ENERGY directory) - - Returns - ------- - views_dict: dict - Dictionary containing view names as keys and a list of the - corresponding variables as values. If the subview_sep is defined, - then the dictionary will have a nested dict containing the subviews. - - """ - # split the sketch in different views - sketch = list(map(lambda x: x.strip(), sketch.split("\\\\\\---/// "))) - - view_elements = {} - for module in sketch: - for sketch_line in module.split("\n"): - # line is a dict with keys "variable_name" and "view_name" - line = parse_sketch_line(sketch_line.strip(), namespace) - - if line["view_name"]: - view_name = line["view_name"] - view_elements[view_name] = [] - - if line["variable_name"]: - if line["variable_name"] not in view_elements[view_name]: - view_elements[view_name].append(line["variable_name"]) - - # removes views that do not include any variable in them - non_empty_views = { - key.lower(): value for key, value in view_elements.items() if value - } - - # split into subviews, if subview_sep is provided - views_dict = {} - if subview_sep and any( - sep in view for sep in subview_sep for view in non_empty_views): - escaped_separators = list(map(lambda x: re.escape(x), subview_sep)) - for full_name, values in non_empty_views.items(): - # split the full view name using the separator and make the - # individual parts safe file or directory names - clean_view_parts = utils.clean_file_names( - *re.split( - "|".join(escaped_separators), - full_name)) - # creating a nested dict for each view.subview - # (e.g. {view_name: {subview_name: [values]}}) - nested_dict = values - - for item in reversed(clean_view_parts): - - nested_dict = {item: nested_dict} - # merging the new nested_dict into the views_dict, preserving - # repeated keys - utils.merge_nested_dicts(views_dict, nested_dict) - - # view names do not have separators or separator characters not provided - else: - if subview_sep and not any( - sep in view for sep in subview_sep for view in non_empty_views): - warnings.warn("The given subview separators were not matched in " - + "any view name.") - - for view_name, elements in non_empty_views.items(): - views_dict[utils.clean_file_names(view_name)[0]] = elements - - return views_dict - - -def _split_sketch(text): - """ - Splits the model file between the main section and the sketch - - Parameters - ---------- - text : string - Full model as a string. - - Returns - ------- - text: string - Model file without sketch. - - sketch: string - Model sketch. - - """ - split_model = text.split("\\\\\\---///", 1) - text = split_model[0] - - try: - sketch = split_model[1] - # remove plots section, if it exists - sketch = sketch.split("///---\\\\\\")[0] - except LookupError: - sketch = "" - warnings.warn("Your model does not have a sketch.") - - return text, sketch - - -def translate_vensim(mdl_file, split_views, encoding=None, **kwargs): - """ - Translate a vensim file. - - Parameters - ---------- - mdl_file: str or pathlib.PosixPath - File path of a vensim model file to translate to python. - - split_views: bool - If True, the sketch is parsed to detect model elements in each - model view, and then translate each view in a separate python - file. Setting this argument to True is recommended for large - models that are split in many different views. - - encoding: str or None (optional) - Encoding of the source model file. If None, the encoding will be - read from the model, if the encoding is not defined in the model - file it will be set to 'UTF-8'. Default is None. - - **kwargs: (optional) - Additional parameters passed to the translate_vensim function - - Returns - ------- - outfile_name: str - Name of the output file. - - Examples - -------- - >>> translate_vensim('teacup.mdl') - - """ - # character used to place subviews in the parent view folder - subview_sep = kwargs.get("subview_sep", "") - - if isinstance(mdl_file, str): - mdl_file = pathlib.Path(mdl_file) - - # check for model extension - if mdl_file.suffix.lower() != ".mdl": - raise ValueError( - "The file to translate, " - + str(mdl_file) - + " is not a vensim model. It must end with mdl extension." - ) - - root_path = mdl_file.parent - - if encoding is None: - encoding = _detect_encoding_from_file(mdl_file) - - with open(mdl_file, "r", encoding=encoding, errors="ignore") as in_file: - text = in_file.read() - - outfile_name = mdl_file.with_suffix(".py") - - if split_views: - text, sketch = _split_sketch(text) - else: - sketch = "" - - file_sections = get_file_sections(text.replace("\n", "")) - - for section in file_sections: - if section["name"] == "_main_": - section["file_path"] = outfile_name - else: # separate macro elements into their own files - section["py_name"] = utils.make_python_identifier( - section["name"]) - section["file_name"] = section["py_name"] + ".py" - section["file_path"] = root_path.joinpath(section["file_name"]) - - macro_list = [s for s in file_sections if s["name"] != "_main_"] - - for section in file_sections: - translate_section(section, macro_list, sketch, root_path, subview_sep) - - return outfile_name - - -def _detect_encoding_from_file(mdl_file): - - try: - with open(mdl_file, "rb") as in_file: - f_line = in_file.readline() - f_line = f_line.decode(detect(f_line)['encoding']) - return re.search(r"(?<={)(.*)(?=})", f_line).group() - except (AttributeError, UnicodeDecodeError): - warnings.warn( - "No encoding specified or detected to translate the model " - "file. 'UTF-8' encoding will be used.") - return "UTF-8" diff --git a/pysd/translation/xmile/SMILE2Py.py b/pysd/translation/xmile/SMILE2Py.py deleted file mode 100644 index 8a789af6..00000000 --- a/pysd/translation/xmile/SMILE2Py.py +++ /dev/null @@ -1,357 +0,0 @@ -""" -Created August 14 2014 -James Houghton - -Changed May 03 2017 -Alexey Prey Mulyukin from sdCloud.io developement team - Changes: - - [May 03 2017] Alexey Prey Mulyukin: Integrate support to - logical operators like 'AND', 'OR' and 'NOT'. - Fix support the whitespaces in expressions between - operators and operands. - Add support to modulo operator - 'MOD'. - Fix support for case insensitive in function names. - -This module converts a string of SMILE syntax into Python - -""" -import parsimonious -from parsimonious.nodes import NodeVisitor -import pkg_resources -import re -from .. import builder, utils - -# Here we define which python function each XMILE keyword corresponds to -functions = { - # === - # 3.5.1 Mathematical Functions - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039980 - # === - - "abs": "abs", - "int": "int", - "inf": {"name": "np.inf", "module": "numpy"}, - "exp": {"name": "np.exp", "module": "numpy"}, - "sin": {"name": "np.sin", "module": "numpy"}, - "cos": {"name": "np.cos", "module": "numpy"}, - "tan": {"name": "np.tan", "module": "numpy"}, - "arcsin": {"name": "np.arcsin", "module": "numpy"}, - "arccos": {"name": "np.arccos", "module": "numpy"}, - "arctan": {"name": "np.arctan", "module": "numpy"}, - "sqrt": {"name": "np.sqrt", "module": "numpy"}, - "ln": {"name": "np.log", "module": "numpy"}, - "log10": {"name": "np.log10", "module": "numpy"}, - "max": "max", - "min": "min", - - # === - # 3.5.2 Statistical Functions - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039981 - # === - - "exprnd": {"name": "np.random.exponential", "module": "numpy"}, - "lognormal": {"name": "np.random.lognormal", "module": "numpy"}, - "normal": {"name": "np.random.normal", "module": "numpy"}, - "poisson": {"name": "np.random.poisson", "module": "numpy"}, - "random": {"name": "np.random.rand", "module": "numpy"}, - - # === - # 3.5.4 Test Input Functions - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039983 - # === - - "pulse": { - "name": "pulse_magnitude", - "parameters": [ - {"name": 'time', "type": "time"}, - {"name": 'magnitude'}, - {"name": 'start'}, - {"name": "repeat_time", "optional": True} - ], - "module": "functions" - }, - "step": { - "name": "step", - "parameters": [ - {"name": 'time', "type": 'time'}, - {"name": 'value'}, - {"name": 'tstep'} - ], - "module": "functions" - }, - # time, slope, start, finish=0 - "ramp": { - "name": "ramp", - "parameters": [ - {"name": 'time', "type": 'time'}, - {"name": 'slope'}, - {"name": 'start'}, - {"name": 'finish', "optional": True} - ], - "module": "functions" - }, - - # === - # 3.5.6 Miscellaneous Functions - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039985 - # === - "if then else": { - "name": "if_then_else", - "parameters": [ - {"name": 'condition'}, - {"name": 'val_if_true', "type": 'lambda'}, - {"name": 'val_if_false', "type": 'lambda'} - ], - "module": "functions" - }, - - # TODO functions/stateful objects to be added - # https://github.com/JamesPHoughton/pysd/issues/154 - "forecast": {"name": "not_implemented_function", "module": "functions", - "original_name": "forecast"}, - "previous": {"name": "not_implemented_function", "module": "functions", - "original_name": "previous"}, - "self": {"name": "not_implemented_function", "module": "functions", - "original_name": "self"} -} - -prefix_operators = { - "not": " not ", - "-": "-", - "+": " ", -} - -infix_operators = { - "and": " and ", - "or": " or ", - "=": "==", - "<=": "<=", - "<": "<", - ">=": ">=", - ">": ">", - "<>": "!=", - "^": "**", - "+": "+", - "-": "-", - "*": "*", - "/": "/", - "mod": "%", -} - -# ==== -# 3.5.3 Delay Functions -# http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039982 -# ==== - -builders = { - # "delay" !TODO! How to add the infinity delay? - - "delay1": lambda element, subscript_dict, args: - builder.add_n_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2] if len(args) > 2 else args[0], - order="1", - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "delay3": lambda element, subscript_dict, args: - builder.add_n_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2] if len(args) > 2 else args[0], - order="3", - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "delayn": lambda element, subscript_dict, args: - builder.add_n_delay( - identifier=element["py_name"], - delay_input=args[0], - delay_time=args[1], - initial_value=args[2] if len(args) > 3 else args[0], - order=args[2], - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "smth1": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2] if len(args) > 2 else args[0], - order="1", - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "smth3": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2] if len(args) > 2 else args[0], - order="3", - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "smthn": lambda element, subscript_dict, args: - builder.add_n_smooth( - identifier=element["py_name"], - smooth_input=args[0], - smooth_time=args[1], - initial_value=args[2] if len(args) > 3 else args[0], - order=args[2], - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - # "forcst" !TODO! - - "trend": lambda element, subscript_dict, args: - builder.add_n_trend( - identifier=element["py_name"], - trend_input=args[0], - average_time=args[1], - initial_trend=args[2] if len(args) > 2 else 0, - subs=element["subs"], - merge_subs=None, - deps=element["dependencies"] - ), - - "init": lambda element, subscript_dict, args: - builder.add_initial( - identifier=element["py_name"], - value=args[0], - deps=element["dependencies"]), -} - - -def format_word_list(word_list): - return '|'.join( - [re.escape(k) for k in reversed(sorted(word_list, key=len))]) - - -class SMILEParser(NodeVisitor): - def __init__(self, model_namespace={}, subscript_dict={}): - - self.model_namespace = model_namespace - self.subscript_dict = subscript_dict - self.extended_model_namespace = { - key.replace(' ', '_'): value - for key, value in self.model_namespace.items()} - self.extended_model_namespace.update(self.model_namespace) - - # === - # 3.5.5 Time Functions - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039984 - # === - self.extended_model_namespace.update({'dt': 'time_step'}) - self.extended_model_namespace.update({'starttime': 'initial_time'}) - self.extended_model_namespace.update({'endtime': 'final_time'}) - - grammar = pkg_resources.resource_string( - "pysd", "translation/xmile/smile.grammar") - grammar = grammar.decode('ascii').format( - funcs=format_word_list(functions.keys()), - in_ops=format_word_list(infix_operators.keys()), - pre_ops=format_word_list(prefix_operators.keys()), - identifiers=format_word_list(self.extended_model_namespace.keys()), - build_keywords=format_word_list(builders.keys()) - ) - - self.grammar = parsimonious.Grammar(grammar) - - def parse(self, text, element, context='eqn'): - """ - context : 'eqn', 'defn' - If context is set to equation, lone identifiers will be - parsed as calls to elements. If context is set to definition, - lone identifiers will be cleaned and returned. - """ - - # Remove the inline comments from `text` before parsing the grammar - # http://docs.oasis-open.org/xmile/xmile/v1.0/csprd01/xmile-v1.0-csprd01.html#_Toc398039973 - text = re.sub(r"\{[^}]*\}", "", text) - if "dependencies" not in element: - element["dependencies"] = dict() - - self.ast = self.grammar.parse(text) - self.context = context - self.element = element - self.new_structure = [] - - py_expr = self.visit(self.ast) - - return ({ - 'py_expr': py_expr - }, self.new_structure) - - def visit_conditional_statement(self, n, vc): - return builder.build_function_call(functions["if then else"], vc[2::4]) - - def visit_user_call_identifier(self, n, vc): - return self.extended_model_namespace[n.text] - - def visit_user_call_quoted_identifier(self, n, vc): - return self.extended_model_namespace[vc[1]] - - def visit_identifier(self, n, vc): - subelement = self.extended_model_namespace[n.text] - utils.update_dependency(subelement, self.element["dependencies"]) - return subelement + '()' - - def visit_quoted_identifier(self, n, vc): - subelement = self.extended_model_namespace[vc[1]] - utils.update_dependency(subelement, self.element["dependencies"]) - return subelement + '()' - - def visit_call(self, n, vc): - function_name = vc[0].lower() - arguments = [e.strip() for e in vc[4].split(",")] - return builder.build_function_call( - functions[function_name], arguments, self.element["dependencies"]) - - def visit_user_call(self, n, vc): - return vc[0] + '(' + vc[4] + ')' - - def visit_build_call(self, n, vc): - builder_name = vc[0].lower() - arguments = [e.strip() for e in vc[4].split(",")] - name, structure = builders[builder_name]( - self.element, self.subscript_dict, arguments) - self.new_structure += structure - self.element["dependencies"] = {structure[-1]["py_name"]: 1} - return name - - def visit_pre_oper(self, n, vc): - return prefix_operators[n.text.lower()] - - def visit_in_oper(self, n, vc): - return infix_operators[n.text.lower()] - - def generic_visit(self, n, vc): - """ - Replace childbearing nodes with a list of their children; - for leaves, return the node text; - for empty nodes, return an empty string. - - Handles: - - call - - parens - - - """ - return ''.join(filter(None, vc)) or n.text or '' diff --git a/pysd/translation/xmile/smile.grammar b/pysd/translation/xmile/smile.grammar deleted file mode 100644 index c6250b3b..00000000 --- a/pysd/translation/xmile/smile.grammar +++ /dev/null @@ -1,25 +0,0 @@ -expr = conditional_statement / (_ pre_oper? _ primary _ (in_oper _ expr)?) - -conditional_statement = "IF" _ expr _ "THEN" _ expr _ "ELSE" _ expr - -primary = call / build_call / user_call / parens / number / identifier / quoted_identifier -parens = "(" _ expr _ ")" -call = func _ "(" _ arguments _ ")" -build_call = build_keyword _ "(" _ arguments _ ")" -user_call = user_call_identifiers _ "(" _ arguments _ ")" -arguments = (expr _ ","? _)* - -number = ((~"[0-9]"+ "."? ~"[0-9]"*) / ("." ~"[0-9]"+)) (("e"/"E") ("-"/"+") ~"[0-9]"+)? - -_ = spacechar* -spacechar = " "* ~"\t"* - -func = ~r"{funcs}"i -build_keyword = ~r"{build_keywords}"i -pre_oper = ~r"{pre_ops}"i -in_oper = ~r"{in_ops}"i -user_call_identifiers = user_call_identifier / user_call_quoted_identifier -user_call_identifier = ~r"{identifiers}"i -user_call_quoted_identifier = "\"" ~r"{identifiers}" "\"" -identifier = ~r"{identifiers}"i -quoted_identifier = "\"" ~r"{identifiers}" "\"" \ No newline at end of file diff --git a/pysd/translation/xmile/xmile2py.py b/pysd/translation/xmile/xmile2py.py deleted file mode 100644 index e714d4ed..00000000 --- a/pysd/translation/xmile/xmile2py.py +++ /dev/null @@ -1,403 +0,0 @@ -""" -Deals with accessing the components of the xmile file, and -formatting them for the builder - -James Houghton -Alexey Prey Mulyukin from sdCloud.io development team. - -""" -import re -import os.path - -from .SMILE2Py import SMILEParser -from lxml import etree -from .. import builder, utils - -import numpy as np - - -def translate_xmile(xmile_file): - """ Translate an xmile model file into a python class. - Functionality is currently limited. - - """ - # process xml file - xml_parser = etree.XMLParser(encoding="utf-8", recover=True) - root = etree.parse(xmile_file, parser=xml_parser).getroot() - NS = root.nsmap[None] # namespace of the xmile document - - def get_xpath_text(node, path, ns=None, default=''): - """ Safe access of occassionally missing elements """ - # defined here to take advantage of NS in default - if ns is None: - ns = {'ns': NS} - try: - return node.xpath(path, namespaces=ns)[0].text - except IndexError: - return default - - def get_xpath_attrib(node, path, attrib, ns=None, default=None): - """ Safe access of occassionally missing elements """ - # defined here to take advantage of NS in default - if ns is None: - ns = {'ns': NS} - try: - return node.xpath(path, namespaces=ns)[0].attrib[attrib] - except IndexError: - return default - - def is_constant_expression(py_expr): - try: - float(py_expr) - return True - except ValueError: - return False - - def parse_lookup_xml_node(node): - ys_node = node.xpath('ns:ypts', namespaces={'ns': NS})[0] - ys = np.fromstring( - ys_node.text, - dtype=float, - sep=ys_node.attrib['sep'] if 'sep' in ys_node.attrib else ',' - ) - xscale_node = node.xpath('ns:xscale', namespaces={'ns': NS}) - if len(xscale_node) > 0: - xmin = xscale_node[0].attrib['min'] - xmax = xscale_node[0].attrib['max'] - xs = np.linspace(float(xmin), float(xmax), len(ys)) - else: - xs_node = node.xpath('ns:xpts', namespaces={'ns': NS})[0] - xs = np.fromstring( - xs_node.text, - dtype=float, - sep=xs_node.attrib['sep'] if 'sep' in xs_node.attrib else ',' - ) - - type = node.attrib['type'] if 'type' in node.attrib else 'continuous' - - functions_map = { - "continuous": { - "name": "lookup", - "module": "functions" - }, - 'extrapolation': { - "name": "lookup_extrapolation", - "module": "functions" - }, - 'discrete': { - "name": "lookup_discrete", - "module": "functions" - } - } - lookup_function = functions_map[type] if type in functions_map\ - else functions_map['continuous'] - - return { - 'name': node.attrib['name'] if 'name' in node.attrib else '', - 'xs': xs, - 'ys': ys, - 'type': type, - 'function': lookup_function - } - - # build model namespace - namespace = { - 'TIME': 'time', - 'Time': 'time', - 'time': 'time' - } # namespace of the python model - names_xpath = '//ns:model/ns:variables/ns:aux|' \ - '//ns:model/ns:variables/ns:flow|' \ - '//ns:model/ns:variables/ns:stock|' \ - '//ns:model/ns:variables/ns:gf' - - for node in root.xpath(names_xpath, namespaces={'ns': NS}): - name = node.attrib['name'] - utils.make_python_identifier(name, namespace) - - model_elements = [] - smile_parser = SMILEParser(namespace) - - # add aux and flow elements - flaux_xpath =\ - '//ns:model/ns:variables/ns:aux|//ns:model/ns:variables/ns:flow' - for node in root.xpath(flaux_xpath, namespaces={'ns': NS}): - name = node.attrib['name'] - units = get_xpath_text(node, 'ns:units') - lims = ( - get_xpath_attrib(node, 'ns:range', 'min'), - get_xpath_attrib(node, 'ns:range', 'max') - ) - lims = str(tuple(float(x) if x is not None else x for x in lims)) - doc = get_xpath_text(node, 'ns:doc') - py_name = namespace[name] - eqn = get_xpath_text(node, 'ns:eqn') - - # Replace new lines with space, and replace 2 or more spaces with - # single space. Then ensure there is no space at start or end of - # equation - eqn = (re.sub(r"(\s{2,})", " ", eqn.replace("\n", ' ')).strip()) - - element = { - 'kind': 'component', - 'real_name': name, - 'unit': units, - 'doc': doc, - 'eqn': eqn, - 'lims': lims, - 'py_name': py_name, - 'subs': [], # Todo later - 'arguments': '', - } - - tranlation, new_structure = smile_parser.parse(eqn, element) - element.update(tranlation) - if is_constant_expression(element['py_expr']): - element['kind'] = 'constant' - - model_elements += new_structure - - gf_node = node.xpath("ns:gf", namespaces={'ns': NS}) - if len(gf_node) > 0: - gf_data = parse_lookup_xml_node(gf_node[0]) - xs = '[' + ','.join("%10.3f" % x for x in gf_data['xs']) + ']' - ys = '[' + ','.join("%10.3f" % x for x in gf_data['ys']) + ']' - py_expr =\ - builder.build_function_call(gf_data['function'], - [element['py_expr'], xs, ys])\ - + ' if x is None else '\ - + builder.build_function_call(gf_data['function'], - ['x', xs, ys]) - element.update({ - 'kind': 'lookup', - # This lookup declared as inline, so we should implement - # inline mode for flow and aux - 'arguments': "x = None", - 'py_expr': py_expr - }) - - model_elements.append(element) - - # add gf elements - gf_xpath = '//ns:model/ns:variables/ns:gf' - for node in root.xpath(gf_xpath, namespaces={'ns': NS}): - name = node.attrib['name'] - py_name = namespace[name] - - units = get_xpath_text(node, 'ns:units') - doc = get_xpath_text(node, 'ns:doc') - - gf_data = parse_lookup_xml_node(node) - xs = '[' + ','.join("%10.3f" % x for x in gf_data['xs']) + ']' - ys = '[' + ','.join("%10.3f" % x for x in gf_data['ys']) + ']' - py_expr = builder.build_function_call(gf_data['function'], - ['x', xs, ys]) - element = { - 'kind': 'lookup', - 'real_name': name, - 'unit': units, - 'lims': None, - 'doc': doc, - 'eqn': '', - 'py_name': py_name, - 'py_expr': py_expr, - 'arguments': 'x', - 'dependencies': {"__lookup__": None}, - 'subs': [], # Todo later - } - model_elements.append(element) - - # add stock elements - stock_xpath = '//ns:model/ns:variables/ns:stock' - for node in root.xpath(stock_xpath, namespaces={'ns': NS}): - name = node.attrib['name'] - units = get_xpath_text(node, 'ns:units') - lims = ( - get_xpath_attrib(node, 'ns:range', 'min'), - get_xpath_attrib(node, 'ns:range', 'max') - ) - lims = str(tuple(float(x) if x is not None else x for x in lims)) - doc = get_xpath_text(node, 'ns:doc') - py_name = namespace[name] - - # Extract input and output flows equations - inflows = [ - n.text for n in node.xpath('ns:inflow', namespaces={'ns': NS})] - outflows = [ - n.text for n in node.xpath('ns:outflow', namespaces={'ns': NS})] - - eqn = ' + '.join(inflows) if inflows else '' - eqn += (' - ' + ' - '.join(outflows)) if outflows else '' - - element = { - 'kind': 'component' if inflows or outflows else 'constant', - 'real_name': name, - 'unit': units, - 'doc': doc, - 'eqn': eqn, - 'lims': lims, - 'py_name': py_name, - 'subs': [], # Todo later - 'arguments': '' - } - - # Parse each flow equations - py_inflows = [] - for inputFlow in inflows: - translation, new_structure = smile_parser.parse( - inputFlow, element) - py_inflows.append(translation['py_expr']) - model_elements += new_structure - - # Parse each flow equations - py_outflows = [] - for outputFlow in outflows: - translation, new_structure = smile_parser.parse( - outputFlow, element) - py_outflows.append(translation['py_expr']) - model_elements += new_structure - - py_ddt = ' + '.join(py_inflows) if py_inflows else '' - py_ddt += (' - ' + ' - '.join(py_outflows)) if py_outflows else '' - - # Read the initial value equation for stock element - initial_value_eqn = get_xpath_text(node, 'ns:eqn') - translation, new_structure = smile_parser.parse( - initial_value_eqn, element) - py_initial_value = translation['py_expr'] - model_elements += new_structure - - py_expr, new_structure = builder.add_stock( - identifier=py_name, - subs=[], # Todo later - merge_subs=[], - expression=py_ddt, - initial_condition=py_initial_value, - deps=element["dependencies"]) - element['py_expr'] = py_expr - element["dependencies"] = {new_structure[-1]["py_name"]: 1} - model_elements.append(element) - model_elements += new_structure - - # remove timestamp pieces so as not to double-count - model_elements_parsed = [] - for element in model_elements: - if element['real_name'].lower() not in ['initial time', - 'final time', - 'time step', - 'saveper']: - model_elements_parsed.append(element) - model_elements = model_elements_parsed - - # Add timeseries information - - # Read the start time of simulation - sim_spec_node = root.xpath('//ns:sim_specs', namespaces={'ns': NS}) - time_units = sim_spec_node[0].attrib['time_units']\ - if len(sim_spec_node) > 0 and 'time_units' in sim_spec_node[0].attrib\ - else "" - - tstart = root.xpath( - '//ns:sim_specs/ns:start', - namespaces={'ns': NS})[0].text - element = { - 'kind': 'constant', - 'real_name': 'INITIAL TIME', - 'unit': time_units, - 'lims': None, - 'doc': 'The initial time for the simulation.', - 'eqn': tstart, - 'py_name': 'initial_time', - 'subs': None, - 'arguments': '', - } - translation, new_structure = smile_parser.parse(tstart, element) - element.update(translation) - model_elements.append(element) - model_elements += new_structure - - # Read the final time of simulation - tstop = root.xpath('//ns:sim_specs/ns:stop', namespaces={'ns': NS})[0].text - element = { - 'kind': 'constant', - 'real_name': 'FINAL TIME', - 'unit': time_units, - 'lims': None, - 'doc': 'The final time for the simulation.', - 'eqn': tstart, - 'py_name': 'final_time', - 'subs': None, - 'arguments': '', - } - - translation, new_structure = smile_parser.parse(tstop, element) - element.update(translation) - model_elements.append(element) - model_elements += new_structure - - # Read the time step of simulation - dt_node = root.xpath('//ns:sim_specs/ns:dt', namespaces={'ns': NS}) - - # Use default value for time step if `dt` is not specified in model - dt_eqn = "1.0" - if len(dt_node) > 0: - dt_node = dt_node[0] - dt_eqn = dt_node.text - # If reciprocal mode are defined for `dt`, we should inverse value - if "reciprocal" in dt_node.attrib\ - and dt_node.attrib["reciprocal"].lower() == "true": - dt_eqn = "1/" + dt_eqn - - element = { - 'kind': 'constant', - 'real_name': 'TIME STEP', - 'unit': time_units, - 'lims': None, - 'doc': 'The time step for the simulation.', - 'eqn': dt_eqn, - 'py_name': 'time_step', - 'subs': None, - 'arguments': '', - } - translation, new_structure = smile_parser.parse(dt_eqn, element) - element.update(translation) - model_elements.append(element) - model_elements += new_structure - - # Add the SAVEPER attribute to the model - model_elements.append({ - 'kind': 'constant', - 'real_name': 'SAVEPER', - 'unit': time_units, - 'lims': None, - 'doc': 'The time step for the simulation.', - 'eqn': dt_eqn, - 'py_name': 'saveper', - 'py_expr': 'time_step()', - 'subs': None, - 'dependencies': {'time_step': 1}, - 'arguments': '', - }) - - # send the pieces to be built - build_elements = builder.merge_partial_elements([ - e for e in model_elements if e["kind"] not in ["subdef", "test", - "section"] - ]) - - dependencies = { - element["py_name"]: element["dependencies"] - - for element in build_elements - if element["dependencies"] is not None - } - file_name, file_extension = os.path.splitext(xmile_file) - outfile_name = file_name + '.py' - - builder.build(elements=build_elements, - subscript_dict={}, - namespace=namespace, - dependencies=dependencies, - outfile_name=outfile_name) - - return outfile_name diff --git a/pysd/translation/xmile/__init__.py b/pysd/translators/__init__.py similarity index 100% rename from pysd/translation/xmile/__init__.py rename to pysd/translators/__init__.py diff --git a/pysd/translators/structures/__init__.py b/pysd/translators/structures/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pysd/translators/structures/abstract_expressions.py b/pysd/translators/structures/abstract_expressions.py new file mode 100644 index 00000000..58a90b4e --- /dev/null +++ b/pysd/translators/structures/abstract_expressions.py @@ -0,0 +1,570 @@ +""" +The following abstract structures are used to build the Abstract Syntax +Tree (AST). In general, there is no hierarchy between them. For example, +an ArithmeticStructure can contain a CallStructure which at the same time +contains another ArithmeticStructure. However, some of them could not be +inside another structures due to the restrictions of the source languages. +For example, the GetConstantsStructure cannot be a part of another structure +because it has to appear after the '=' sign in Vensim and not be followed by +anything else. +""" +from dataclasses import dataclass +from typing import Union + + +class AbstractSyntax: + """ + Generic class. All Abstract Synax structured are childs of that class. + Used for typing. + """ + pass + + +@dataclass +class ArithmeticStructure(AbstractSyntax): + """ + Dataclass for an arithmetic structure. + + Parameters + ---------- + operators: list + List of operators applied between the arguments + arguments: list + The arguments of the arithmetics operations. + + """ + operators: list + arguments: list + + def __str__(self) -> str: # pragma: no cover + return "ArithmeticStructure:\n\t %s %s" % ( + self.operators, self.arguments) + + +@dataclass +class LogicStructure(AbstractSyntax): + """ + Dataclass for a logic structure. + + Parameters + ---------- + operators: list + List of operators applied between the arguments + arguments: list + The arguments of the logic operations. + + """ + operators: list + arguments: list + + def __str__(self) -> str: # pragma: no cover + return "LogicStructure:\n\t %s %s" % ( + self.operators, self.arguments) + + +@dataclass +class SubscriptsReferenceStructure(AbstractSyntax): + """ + Dataclass for a subscript reference structure. + + Parameters + ---------- + subscripts: tuple + The list of subscripts referenced. + + """ + subscripts: tuple + + def __str__(self) -> str: # pragma: no cover + return "SubscriptReferenceStructure:\n\t %s" % self.subscripts + + +@dataclass +class ReferenceStructure(AbstractSyntax): + """ + Dataclass for an element reference structure. + + Parameters + ---------- + reference: str + The name of the referenced element. + subscripts: SubscriptsReferenceStructure or None + The subscrips used in the reference. + + """ + reference: str + subscripts: Union[SubscriptsReferenceStructure, None] = None + + def __str__(self) -> str: # pragma: no cover + return "ReferenceStructure:\n\t %s%s" % ( + self.reference, + "\n\t" + str(self.subscripts or "").replace("\n", "\n\t")) + + +@dataclass +class CallStructure(AbstractSyntax): + """ + Dataclass for a call structure. + + Parameters + ---------- + function: str or ReferenceStructure + The name or the reference of the callable. + arguments: tuple + The list of arguments used for calling the function. + + """ + function: Union[str, ReferenceStructure] + arguments: tuple + + def __str__(self) -> str: # pragma: no cover + return "CallStructure:\n\t%s(%s)" % ( + self.function, + "\n\t\t,".join([ + "\n\t\t" + str(arg).replace("\n", "\n\t\t") + for arg in self.arguments + ])) + + +@dataclass +class GameStructure(AbstractSyntax): + """ + Dataclass for a game structure. + + Parameters + ---------- + expression: AST + The expression inside the game call. + + """ + expression: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "GameStructure:\n\t%s" % self.expression + + +@dataclass +class InitialStructure(AbstractSyntax): + """ + Dataclass for a initial structure. + + Parameters + ---------- + initial: AST + The expression inside the initial call. + + """ + initial: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "InitialStructure:\n\t%s" % ( + self.initial) + + +@dataclass +class IntegStructure(AbstractSyntax): + """ + Dataclass for an integ/stock structure. + + Parameters + ---------- + flow: AST + The flow of the stock. + initial: AST + The initial value of the stock. + + """ + flow: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "IntegStructure:\n\t%s,\n\t%s" % ( + self.flow, + self.initial) + + +@dataclass +class DelayStructure(AbstractSyntax): + """ + Dataclass for a delay structure. + + Parameters + ---------- + input: AST + The input of the delay. + delay_time: AST + The delay time value of the delay. + initial: AST + The initial value of the delay. + order: float + The order of the delay. + + """ + input: Union[AbstractSyntax, float] + delay_time: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + order: float + + def __str__(self) -> str: # pragma: no cover + return "DelayStructure (order %s):\n\t%s,\n\t%s,\n\t%s" % ( + self.order, + self.input, + self.delay_time, + self.initial) + + +@dataclass +class DelayNStructure(AbstractSyntax): + """ + Dataclass for a delay n structure. + + Parameters + ---------- + input: AST + The input of the delay. + delay_time: AST + The delay time value of the delay. + initial: AST + The initial value of the delay. + order: float + The order of the delay. + + """ + input: Union[AbstractSyntax, float] + delay_time: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + order: Union[AbstractSyntax, float] + + # DELAY N may behave different than other delays when the delay time + # changes during integration + + def __str__(self) -> str: # pragma: no cover + return "DelayNStructure (order %s):\n\t%s,\n\t%s,\n\t%s" % ( + self.order, + self.input, + self.delay_time, + self.initial) + + +@dataclass +class DelayFixedStructure(AbstractSyntax): + """ + Dataclass for a delay fixed structure. + + Parameters + ---------- + input: AST + The input of the delay. + delay_time: AST + The delay time value of the delay. + initial: AST + The initial value of the delay. + + """ + input: Union[AbstractSyntax, float] + delay_time: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "DelayFixedStructure:\n\t%s,\n\t%s,\n\t%s" % ( + self.input, + self.delay_time, + self.initial) + + +@dataclass +class SmoothStructure(AbstractSyntax): + """ + Dataclass for a smooth structure. + + Parameters + ---------- + input: AST + The input of the smooth. + delay_time: AST + The smooth time value of the smooth. + initial: AST + The initial value of the smooth. + order: float + The order of the smooth. + + """ + input: Union[AbstractSyntax, float] + smooth_time: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + order: float + + def __str__(self) -> str: # pragma: no cover + return "SmoothStructure (order %s):\n\t%s,\n\t%s,\n\t%s" % ( + self.order, + self.input, + self.smooth_time, + self.initial) + + +@dataclass +class SmoothNStructure(AbstractSyntax): + """ + Dataclass for a smooth n structure. + + Parameters + ---------- + input: AST + The input of the smooth. + delay_time: AST + The smooth time value of the smooth. + initial: AST + The initial value of the smooth. + order: float + The order of the smooth. + + """ + input: Union[AbstractSyntax, float] + smooth_time: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + order: Union[AbstractSyntax, float] + + # SMOOTH N may behave different than other smooths with RungeKutta + # integration + + def __str__(self) -> str: # pragma: no cover + return "SmoothNStructure (order %s):\n\t%s,\n\t%s,\n\t%s" % ( + self.order, + self.input, + self.smooth_time, + self.initial) + + +@dataclass +class TrendStructure(AbstractSyntax): + """ + Dataclass for a trend structure. + + Parameters + ---------- + input: AST + The input of the trend. + average_time: AST + The average time value of the trend. + initial_trend: AST + The initial trend value of the trend. + + """ + input: Union[AbstractSyntax, float] + average_time: Union[AbstractSyntax, float] + initial_trend: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "TrendStructure:\n\t%s,\n\t%s,\n\t%s" % ( + self.input, + self.average_time, + self.initial) + + +@dataclass +class ForecastStructure(AbstractSyntax): + """ + Dataclass for a forecast structure. + + Parameters + ---------- + input: AST + The input of the forecast. + averae_time: AST + The average time value of the forecast. + horizon: float + The horizon value of the forecast. + initial_trend: AST + The initial trend value of the forecast. + + """ + input: Union[AbstractSyntax, float] + average_time: Union[AbstractSyntax, float] + horizon: Union[AbstractSyntax, float] + initial_trend: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "ForecastStructure:\n\t%s,\n\t%s,\n\t%s,\n\t%s" % ( + self.input, + self.average_time, + self.horizon, + self.initial_trend) + + +@dataclass +class SampleIfTrueStructure(AbstractSyntax): + """ + Dataclass for a sample if true structure. + + Parameters + ---------- + condition: AST + The condition of the sample if true + input: AST + The input of the sample if true. + initial: AST + The initial value of the sample if true. + + """ + condition: Union[AbstractSyntax, float] + input: Union[AbstractSyntax, float] + initial: Union[AbstractSyntax, float] + + def __str__(self) -> str: # pragma: no cover + return "SampleIfTrueStructure:\n\t%s,\n\t%s,\n\t%s" % ( + self.condition, + self.input, + self.initial) + + +@dataclass +class LookupsStructure(AbstractSyntax): + """ + Dataclass for a lookup structure. + + Parameters + ---------- + x: tuple + The list of the x values of the lookup. + y: tuple + The list of the y values of the lookup. + x_limits: tuple + The minimum and maximum value of x. + y_limits: tuple + The minimum and maximum value of y. + type: str + The interpolation method. + + """ + x: tuple + y: tuple + x_limits: tuple + y_limits: tuple + type: str + + def __str__(self) -> str: # pragma: no cover + return "LookupStructure (%s):\n\tx %s = %s\n\ty %s = %s\n" % ( + self.type, self.x_limits, self.x, self.y_limits, self.y + ) + + +@dataclass +class InlineLookupsStructure(AbstractSyntax): + """ + Dataclass for an inline lookup structure. + + Parameters + ---------- + argument: AST + The argument of the inline lookup. + lookups: LookupStructure + The lookups definition. + + """ + argument: Union[AbstractSyntax, float] + lookups: LookupsStructure + + def __str__(self) -> str: # pragma: no cover + return "InlineLookupsStructure:\n\t%s\n\t%s" % ( + str(self.argument).replace("\n", "\n\t"), + str(self.lookups).replace("\n", "\n\t") + ) + + +@dataclass +class DataStructure(AbstractSyntax): + """ + Dataclass for an empty data structure. + + Parameters + ---------- + None + + """ + pass + + def __str__(self) -> str: # pragma: no cover + return "DataStructure" + + +@dataclass +class GetLookupsStructure(AbstractSyntax): + """ + Dataclass for a get lookups structure. + + Parameters + ---------- + file: str + The file path where the data is. + tab: str + The sheetname where the data is. + x_row_or_col: str + The pointer to the cell or cellrange name that defines the + interpolation series data. + cell: str + The pointer to the cell or the cellrange name that defines the data. + + """ + file: str + tab: str + x_row_or_col: str + cell: str + + def __str__(self) -> str: # pragma: no cover + return "GetLookupStructure:\n\t'%s', '%s', '%s', '%s'\n" % ( + self.file, self.tab, self.x_row_or_col, self.cell + ) + + +@dataclass +class GetDataStructure(AbstractSyntax): + """ + Dataclass for a get lookups structure. + + Parameters + ---------- + file: str + The file path where the data is. + tab: str + The sheetname where the data is. + time_row_or_col: str + The pointer to the cell or cellrange name that defines the + interpolation time series data. + cell: str + The pointer to the cell or the cellrange name that defines the data. + + """ + file: str + tab: str + time_row_or_col: str + cell: str + + def __str__(self) -> str: # pragma: no cover + return "GetDataStructure:\n\t'%s', '%s', '%s', '%s'\n" % ( + self.file, self.tab, self.time_row_or_col, self.cell + ) + + +@dataclass +class GetConstantsStructure(AbstractSyntax): + """ + Dataclass for a get lookups structure. + + Parameters + ---------- + file: str + The file path where the data is. + tab: str + The sheetname where the data is. + cell: str + The pointer to the cell or the cellrange name that defines the data. + + """ + file: str + tab: str + cell: str + + def __str__(self) -> str: # pragma: no cover + return "GetConstantsStructure:\n\t'%s', '%s', '%s'\n" % ( + self.file, self.tab, self.cell + ) diff --git a/pysd/translators/structures/abstract_model.py b/pysd/translators/structures/abstract_model.py new file mode 100644 index 00000000..da0acb84 --- /dev/null +++ b/pysd/translators/structures/abstract_model.py @@ -0,0 +1,402 @@ +""" +The main Abstract dataclasses provide the structure for the information +from the Component level to the Model level. This classes are hierarchical +An AbstractComponent will be inside an AbstractElement, which is inside an +AbstractSection, which is a part of an AbstractModel. + +""" +from dataclasses import dataclass +from typing import Tuple, List, Union +from pathlib import Path + + +@dataclass +class AbstractComponent: + """ + Dataclass for a regular component. + + Parameters + ---------- + subscripts: tuple + Tuple of length two with first argument the list of subscripts + in the variable definition and the second argument the list of + subscripts list that must be ignored (EXCEPT). + ast: object + The AbstractSyntaxTree of the component expression + type: str (optional) + The type of component. 'Auxiliary' by default. + subtype: str (optional) + The subtype of component. 'Normal' by default. + + """ + subscripts: Tuple[List[str], List[List[str]]] + ast: object + type: str = "Auxiliary" + subtype: str = "Normal" + + def __str__(self) -> str: # pragma: no cover + return "AbstractComponent %s\n" % ( + "%s" % repr(list(self.subscripts)) if self.subscripts else "") + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the component to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + if depth == 0: + return self.__str__() + + return self.__str__() + "\n" + self._str_child(depth, indent) + + def _str_child(self, depth, indent) -> str: # pragma: no cover + return str(self.ast).replace("\t", indent).replace("\n", "\n" + indent) + + +@dataclass +class AbstractUnchangeableConstant(AbstractComponent): + """ + Dataclass for an unchangeable constant component. This class is a child + of AbstractComponent. + + Parameters + ---------- + subscripts: tuple + Tuple of length two with first argument the list of subscripts + in the variable definition and the second argument the list of + subscripts list that must be ignored (EXCEPT). + ast: object + The AbstractSyntaxTree of the component expression + type: str (optional) + The type of component. 'Constant' by default. + subtype: str (optional) + The subtype of component. 'Unchangeable' by default. + + """ + subscripts: Tuple[List[str], List[List[str]]] + ast: object + type: str = "Constant" + subtype: str = "Unchangeable" + + def __str__(self) -> str: # pragma: no cover + return "AbstractLookup %s\n" % ( + "%s" % repr(list(self.subscripts)) if self.subscripts else "") + + +@dataclass +class AbstractLookup(AbstractComponent): + """ + Dataclass for a lookup component. This class is a child of + AbstractComponent. + + Parameters + ---------- + subscripts: tuple + Tuple of length two with first argument the list of subscripts + in the variable definition and the second argument the list of + subscripts list that must be ignored (EXCEPT). + ast: object + The AbstractSyntaxTree of the component expression + arguments: str (optional) + The name of the argument to use. 'x' by default. + type: str (optional) + The type of component. 'Lookup' by default. + subtype: str (optional) + The subtype of component. 'Hardcoded' by default. + + """ + subscripts: Tuple[List[str], List[List[str]]] + ast: object + arguments: str = "x" + type: str = "Lookup" + subtype: str = "Hardcoded" + + def __str__(self) -> str: # pragma: no cover + return "AbstractLookup %s\n" % ( + "%s" % repr(list(self.subscripts)) if self.subscripts else "") + + +@dataclass +class AbstractData(AbstractComponent): + """ + Dataclass for a data component. This class is a child + of AbstractComponent. + + Parameters + ---------- + subscripts: tuple + Tuple of length two with first argument the list of subscripts + in the variable definition and the second argument the list of + subscripts list that must be ignored (EXCEPT). + ast: object + The AbstractSyntaxTree of the component expression + keyword: str or None (optional) + The data object keyword ('interpolate', 'hold_backward', + 'look_forward', 'raw'). Default is None. + type: str (optional) + The type of component. 'Data' by default. + subtype: str (optional) + The subtype of component. 'Normal' by default. + + """ + subscripts: Tuple[List[str], List[List[str]]] + ast: object + keyword: Union[str, None] = None + type: str = "Data" + subtype: str = "Normal" + + def __str__(self) -> str: # pragma: no cover + return "AbstractData (%s) %s\n" % ( + self.keyword, + "%s" % repr(list(self.subscripts)) if self.subscripts else "") + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the component to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + if depth == 0: + return self.__str__() + + return self.__str__() + "\n" + self._str_child(depth, indent) + + def _str_child(self, depth, indent) -> str: # pragma: no cover + return str(self.ast).replace("\n", "\n" + indent) + + +@dataclass +class AbstractElement: + """ + Dataclass for an element. + + Parameters + ---------- + name: str + The name of the element. + components: list + The list of AbstractComponents that define this element. + units: str (optional) + The units of the element. '' by default. + limits: tuple (optional) + The limits of the element. (None, None) by default. + units: str (optional) + The documentation of the element. '' by default. + + """ + name: str + components: List[AbstractComponent] + units: str = "" + limits: tuple = (None, None) + documentation: str = "" + + def __str__(self) -> str: # pragma: no cover + return "AbstractElement:\t%s (%s, %s)\n%s\n" % ( + self.name, self.units, self.limits, self.documentation) + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the element to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + if depth == 0: + return self.__str__() + elif depth is not None: + depth -= 1 + + return self.__str__() + "\n" + self._str_child(depth, indent) + + def _str_child(self, depth, indent) -> str: # pragma: no cover + return "\n".join([ + component.dump(depth, indent) for component in self.components + ]).replace("\n", "\n" + indent) + + +@dataclass +class AbstractSubscriptRange: + """ + Dataclass for a subscript range. + + Parameters + ---------- + name: str + The name of the element. + subscripts: list or str or dict + The subscripts as a list of strings for a regular definition, + str for a copy definition and as a dict for a GET XLS/DIRECT + definition. + mapping: list + The list of subscript range that can be mapped to. + + """ + name: str + subscripts: Union[list, str, dict] + mapping: list + + def __str__(self) -> str: # pragma: no cover + return "AbstractSubscriptRange:\t%s\n\t%s\n" % ( + self.name, + "%s <- %s" % (self.subscripts, self.mapping) + if self.mapping else self.subscripts) + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the subscript range to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + return self.__str__() + + +@dataclass +class AbstractSection: + """ + Dataclass for an element. + + Parameters + ---------- + name: str + Section name. '__main__' for the main section or the macro name. + path: pathlib.Path + Section path. It should be the model name for main section and + the clean macro name for a macro. + section_type: str ('main' or 'macro') + The section type. + params: list + List of params that takes the section. In the case of main + section it will be an empty list. + returns: list + List of variables that returns the section. In the case of main + section it will be an empty list. + subscripts: tuple + Tuple of AbstractSubscriptRanges that are defined in the section. + elements: tuple + Tuple of AbstractElements that are defined in the section. + split: bool + If split is True the created section will split the variables + depending on the views_dict. + views_dict: dict + The dictionary of the views. Giving the variables classified at + any level in order to split them by files. + + """ + name: str + path: Path + type: str # main, macro or module + params: List[str] + returns: List[str] + subscripts: Tuple[AbstractSubscriptRange] + elements: Tuple[AbstractElement] + split: bool + views_dict: Union[dict, None] + + def __str__(self) -> str: # pragma: no cover + return "AbstractSection (%s):\t%s (%s)\n" % ( + self.type, self.name, self.path) + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the section to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + if depth == 0: + return self.__str__() + elif depth is not None: + depth -= 1 + + return self.__str__() + "\n" + self._str_child(depth, indent) + + def _str_child(self, depth, indent) -> str: # pragma: no cover + return "\n".join([ + element.dump(depth, indent) for element in self.subscripts + ] + [ + element.dump(depth, indent) for element in self.elements + ]).replace("\n", "\n" + indent) + + +@dataclass +class AbstractModel: + """ + Dataclass for an element. + + Parameters + ---------- + original_path: pathlib.Path + The path to the original file. + sections: tuple + Tuple of AbstractSectionss that are defined in the model. + + """ + original_path: Path + sections: Tuple[AbstractSection] + + def __str__(self) -> str: # pragma: no cover + return "AbstractModel:\t%s\n" % self.original_path + + def dump(self, depth=None, indent="") -> str: # pragma: no cover + """ + Dump the model to a printable version. + + Parameters + ---------- + depth: int (optional) + The number of depht levels to show in the dumped output. + Default is None which will dump everything. + + indent: str (optional) + The indent to use for a lower level object. Default is ''. + + """ + if depth == 0: + return self.__str__() + elif depth is not None: + depth -= 1 + + return self.__str__() + "\n" + self._str_child(depth, indent) + + def _str_child(self, depth, indent) -> str: # pragma: no cover + return "\n".join([ + section.dump(depth, indent) for section in self.sections + ]).replace("\n", "\n" + indent) diff --git a/pysd/translators/vensim/__init__.py b/pysd/translators/vensim/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pysd/translators/vensim/parsing_grammars/common_grammar.peg b/pysd/translators/vensim/parsing_grammars/common_grammar.peg new file mode 100644 index 00000000..6b6e5f8c --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/common_grammar.peg @@ -0,0 +1,19 @@ +# Parsing Expression Grammar: common_grammar + +name = basic_id / escape_group + +# This takes care of models with Unicode variable names +basic_id = id_start id_continue* + +id_start = ~r"[\w]"IU +id_continue = id_start / ~r"[0-9\'\$\s\_]" + +# between quotes, either escaped quote or character that is not a quote +escape_group = "\"" ( "\\\"" / ~r"[^\"]" )* "\"" + +number = raw_number +raw_number = ("+"/"-")? _ (~r"\d+\.?\d*([eE][+-]?\d+)?" / ~r"\.\d+([eE][+-]?\d+)?") +string = "\'" (~r"[^\']"IU)* "\'" +limits = _ "[" ~r"[^\]]*" "]" _ "," + +_ = ~r"[\s\\]*" diff --git a/pysd/translators/vensim/parsing_grammars/components.peg b/pysd/translators/vensim/parsing_grammars/components.peg new file mode 100644 index 00000000..927a4988 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/components.peg @@ -0,0 +1,34 @@ +# Parsing Expression Grammar: components + +expr_type = array / final_expr + +final_expr = logic_expr (_ logic_oper _ logic_expr)* # logic operators (:and:, :or:) +logic_expr = not_oper? _ comp_expr # :not: operator +comp_expr = add_expr (_ comp_oper _ add_expr)? # comparison (e.g. '<', '=>') +add_expr = prod_expr (_ add_oper _ prod_expr)* # addition and substraction +prod_expr = exp_expr (_ prod_oper _ exp_expr)* # product and division +exp_expr = neg_expr (_ exp_oper _ neg_expr)* # exponential +neg_expr = pre_oper? _ expr # pre operators (-, +) +expr = lookup_with_def / call / parens / number / reference / nan + +lookup_with_def = ~r"(WITH\ LOOKUP)"I _ "(" _ final_expr _ "," _ "(" _ limits? ( _ "(" _ raw_number _ "," _ raw_number _ ")" _ ","? _ )+ _ ")" _ ")" + +nan = ":NA:" + +arguments = ((string / final_expr) _ ","? _)* +parens = "(" _ final_expr _ ")" + +call = reference _ "(" _ arguments _ ")" + +reference = (name _ subscript_list) / name # check first for subscript +subscript_list = "[" _ (name _ "!"? _ ","? _)+ _ "]" + +array = (raw_number _ ("," / ";")? _)+ !~r"." # negative lookahead for + +logic_oper = ~r"(%(logic_ops)s)"IU +not_oper = ~r"(%(not_ops)s)"IU +comp_oper = ~r"(%(comp_ops)s)"IU +add_oper = ~r"(%(add_ops)s)"IU +prod_oper = ~r"(%(prod_ops)s)"IU +exp_oper = ~r"(%(exp_ops)s)"IU +pre_oper = ~r"(%(pre_ops)s)"IU diff --git a/pysd/translators/vensim/parsing_grammars/element_object.peg b/pysd/translators/vensim/parsing_grammars/element_object.peg new file mode 100644 index 00000000..49256520 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/element_object.peg @@ -0,0 +1,48 @@ +# Parsing Expression Grammar: element_object + +entry = unchangeable_constant / component / data_definition / subscript_definition / lookup_definition / subscript_copy + +# Regular component definition "=" +component = name _ subscript_component? _ "=" _ expression + +# Unchangeable constant definition "==" +unchangeable_constant = name _ subscript_component? _ "==" _ expression + +# Lookup definition "()", uses lookahead assertion to capture whole group +lookup_definition = name _ subscript_component? &"(" _ expression + + +# Data type definition ":=" or empty with keyword +data_definition = component_data_definition / empty_data_definition +component_data_definition = name _ subscript_component? _ keyword? _ ":=" _ expression +empty_data_definition = name _ subscript_component? _ keyword + +# Subscript ranges +# Subcript range regular definition ":" +subscript_definition = name _ ":" _ (imported_subscript / literal_subscript) _ subscript_mapping_list? +imported_subscript = basic_id _ "(" _ (string _ ","? _)* ")" +literal_subscript = (subscript_range / subscript) _ ("," _ (subscript_range / subscript) _)* +subscript_range = "(" _ basic_id _ "-" _ basic_id _ ")" + +# Subcript range definition by copy "<->" +subscript_copy = name _ "<->" _ name_mapping + +# Subscript mapping +subscript_mapping_list = "->" _ subscript_mapping _ ("," _ subscript_mapping _)* +subscript_mapping = (_ name_mapping _) / (_ "(" _ name_mapping _ ":" _ index_list _")" ) +name_mapping = basic_id / escape_group + +# Subscript except match +subscript_list_except = ":EXCEPT:" _ subscript_except_group (_ ',' _ subscript_except_group)* +subscript_except_group = '[' _ subscript_except _ ("," _ subscript_except _)* _ ']' +subscript_except = basic_id / escape_group + +# Subscript match +subscript_list = "[" _ index_list _ "]" +index_list = subscript _ ("," _ subscript _)* +subscript = basic_id / escape_group + +# Other definitions +subscript_component = subscript_list _ subscript_list_except? +expression = ~r".*" # expression could be anything, at this point. +keyword = ":" _ basic_id _ ":" diff --git a/pysd/translators/vensim/parsing_grammars/file_sections.peg b/pysd/translators/vensim/parsing_grammars/file_sections.peg new file mode 100644 index 00000000..2dd9c463 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/file_sections.peg @@ -0,0 +1,15 @@ +# Parsing Expression Grammar: file_sections + +# full file +file = encoding? _ ((macro / main) _)+ + +# macro definition +macro = ":MACRO:" _ name _ "(" _ (name _ ","? _)+ _ ":"? _ (name _ ","? _)* _ ")" ~r".+?(?=:END OF MACRO:)" ":END OF MACRO:" + +# regular expressions +main = main_part / main_end +main_part = !":MACRO:" ~r".+(?=:MACRO:)" +main_end = !":MACRO:" ~r".+" + +# encoding +encoding = ~r"\{[^\}]*\}" diff --git a/pysd/translators/vensim/parsing_grammars/lookups.peg b/pysd/translators/vensim/parsing_grammars/lookups.peg new file mode 100644 index 00000000..c4a731b8 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/lookups.peg @@ -0,0 +1,7 @@ +# Parsing Expression Grammar: lookups + +lookup = _ "(" _ (regularLookup / excelLookup) _ ")" +regularLookup = limits? _ ( "(" _ number _ "," _ number _ ")" _ ","? _ )+ +excelLookup = ~"GET( |_)(XLS|DIRECT)( |_)LOOKUPS"I _ "(" (args _ ","? _)+ ")" + +args = ~r"[^,()]*" diff --git a/pysd/translators/vensim/parsing_grammars/section_elements.peg b/pysd/translators/vensim/parsing_grammars/section_elements.peg new file mode 100644 index 00000000..40250fa5 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/section_elements.peg @@ -0,0 +1,12 @@ +# Parsing Expression Grammar: section_elements + +model = (entry / section)+ sketch? +entry = element "~" element "~" doc ("~" element)? "|" +section = element "~" element "|" +sketch = ~r".*" #anything + +# Either an escape group, or a character that is not tilde or pipe +element = ( escape_group / ~r"[^~|]")* + +# Anything other that is not a tilde or pipe +doc = (~r"[^~|]")* diff --git a/pysd/translators/vensim/parsing_grammars/sketch.peg b/pysd/translators/vensim/parsing_grammars/sketch.peg new file mode 100644 index 00000000..b4dd0546 --- /dev/null +++ b/pysd/translators/vensim/parsing_grammars/sketch.peg @@ -0,0 +1,58 @@ +# Parsing Expression Grammar: sketch + +line = var_definition / view_intro / view_title / view_definition / arrow / flow / other_objects / anything +view_intro = ~r"\s*Sketch.*?names$" / ~r"^V300.*?ignored$" +view_title = "*" view_name +view_name = ~r"(?<=\*)[^\n]+$" +view_definition = "$" color "," digit "," font_properties "|" ( ( color / ones_and_dashes ) "|")* view_code +var_definition = var_code "," var_number "," var_name "," position "," var_box_type "," arrows_in_allowed "," hide_level "," var_face "," var_word_position "," var_thickness "," var_rest_conf ","? ( ( ones_and_dashes / color) ",")* font_properties? ","? extra_bytes? + +# elements used in a line defining the properties of a variable or stock +var_name = element +var_name = ~r"(?<=,)[^,]+(?=,)" +var_number = digit +var_box_type = ~r"(?<=,)\d+,\d+,\d+(?=,)" # improve this regex +arrows_in_allowed = ~r"(?<=,)\d+(?=,)" # if this is an even number it's a shadow variable +hide_level = digit +var_face = digit +var_word_position = ~r"(?<=,)\-*\d+(?=,)" +var_thickness = digit +var_rest_conf = digit "," ~r"\d+" +extra_bytes = ~r"\d+,\d+,\d+,\d+,\d+,\d+" # required since Vensim 8.2.1 +arrow = arrow_code "," digit "," origin_var "," destination_var "," (digit ",")+ (ones_and_dashes ",")? ((color ",") / ("," ~r"\d+") / (font_properties "," ~r"\d+"))* "|(" position ")|" + +# arrow origin and destination (this may be useful if further parsing is required) +origin_var = digit +destination_var = digit + +# flow arrows +flow = source_or_sink_or_plot / flow_arrow + +# if you want to extend the parsing, these three would be a good starting point (they are followed by "anything") +source_or_sink_or_plot = multipurpose_code "," anything +flow_arrow = flow_arrow_code "," anything +other_objects = other_objects_code "," anything + +# fonts +font_properties = font_name? "|" font_size "|" font_style? "|" color +font_style = "B" / "I" / "U" / "S" / "V" # italics, bold, underline, etc +font_size = ~r"\d+" # this needs to be made a regex to match any font +font_name = ~r"(?<=,)[^\|\d]+(?=\|)" + +# x and y within the view layout. This may be useful if further parsing is required +position = ~r"-*\d+,-*\d+" + +# rgb color (e.g. 255-255-255) +color = ~r"((? str: # pragma: no cover + """Get element information.""" + return self.__str__() + + @property + def verbose(self): # pragma: no cover + """Print element information to standard output.""" + print(self._verbose) + + def _parse_units(self, units_str: str) -> Tuple[str, tuple]: + """Separate the limits from the units.""" + # TODO improve units parsing: parse them when parsing the section + # elements + if not units_str: + return "", None + + if units_str.endswith("]"): + units, lims = units_str.rsplit("[") # types: str, str + else: + return units_str, None + + lims = tuple( + [ + float(x) if x.strip() != "?" else None + for x in lims.strip("]").split(",") + ] + ) + return units.strip(), lims + + def parse(self) -> object: + """ + Parse an Element object with parsimonious using the grammar given in + 'parsing_grammars/element_object.peg' and the class + ElementsComponentVisitor to visit the parsed expressions. + + Splits the LHS from the RHS of the equation. If the returned + object is a SubscriptRange, no more parsing is needed. Otherwise, + the RHS of the returned object (Component) should be parsed + to get the AbstractSyntax Tree. + + Returns + ------- + self.component: SubscriptRange or Component + The subscript range definition object or component object. + + """ + tree = vu.Grammar.get("element_object").parse(self.equation) + self.component = ElementsComponentVisitor(tree).component + self.component.units = self.units + self.component.limits = self.limits + self.component.documentation = self.documentation + return self.component + + +class ElementsComponentVisitor(parsimonious.NodeVisitor): + """Visit model element definition to get the component object.""" + + def __init__(self, ast): + self.mapping = [] + self.subscripts = [] + self.subscripts_except = [] + self.subscripts_except_groups = [] + self.name = None + self.expression = None + self.keyword = None + self.visit(ast) + + def visit_subscript_definition(self, n, vc): + self.component = SubscriptRange( + self.name, self.subscripts, self.mapping) + + def visit_lookup_definition(self, n, vc): + self.component = Lookup( + self.name, + (self.subscripts, self.subscripts_except_groups), + self.expression + ) + + def visit_unchangeable_constant(self, n, vc): + self.component = UnchangeableConstant( + self.name, + (self.subscripts, self.subscripts_except_groups), + self.expression + ) + + def visit_component(self, n, vc): + self.component = Component( + self.name, + (self.subscripts, self.subscripts_except_groups), + self.expression + ) + + def visit_data_definition(self, n, vc): + self.component = Data( + self.name, + (self.subscripts, self.subscripts_except_groups), + self.keyword, + self.expression + ) + + def visit_keyword(self, n, vc): + self.keyword = n.text.strip()[1:-1].lower().replace(" ", "_") + + def visit_imported_subscript(self, n, vc): + self.subscripts = { + arg_name: argument.strip().strip("'") + for arg_name, argument + in zip( + ("file", "tab", "firstcell", "lastcell", "prefix"), + vc[4].split(",") + ) + } + + def visit_subscript_copy(self, n, vc): + self.component = SubscriptRange(self.name, vc[4].strip()) + + def visit_subscript_mapping(self, n, vc): + if ":" in str(vc): + # TODO: ensure the correct working of this condition adding + # full integration tests + warnings.warn( + "\nSubscript mapping detected. " + + "This feature works only for simple cases." + ) + # Obtain subscript name and split by : and ( + self.mapping.append(str(vc).split(":")[0].split("(")[1].strip()) + else: + self.mapping.append(vc[0].strip()) + + def visit_subscript_range(self, n, vc): + subs_start = re.findall(r"\d+|\D+", vc[2].strip()) + subs_end = re.findall(r"\d+|\D+", vc[6].strip()) + prefix_start, num_start = "".join(subs_start[:-1]), int(subs_start[-1]) + prefix_end, num_end = "".join(subs_end[:-1]), int(subs_end[-1]) + + if not prefix_start or not prefix_end: + raise ValueError( + "\nA numeric range must contain at least one letter.") + elif num_start >= num_end: + raise ValueError( + "\nThe number of the first subscript value must be " + "lower than the second subscript value in a " + "subscript numeric range.") + elif prefix_start != prefix_end: + raise ValueError( + "\nOnly matching names ending in numbers are valid.") + + self.subscripts += [ + prefix_start + str(i) for i in range(num_start, num_end + 1) + ] + + def visit_name(self, n, vc): + self.name = vc[0].strip() + + def visit_subscript(self, n, vc): + self.subscripts.append(n.text.strip()) + + def visit_subscript_except(self, n, vc): + self.subscripts_except.append(n.text.strip()) + + def visit_subscript_except_group(self, n, vc): + self.subscripts_except_groups.append(self.subscripts_except.copy()) + self.subscripts_except = [] + + def visit_expression(self, n, vc): + self.expression = n.text.strip() + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text + + +class SubscriptRange(): + """ + Subscript range definition, defined by ":" or "<->" in Vensim. + """ + + def __init__(self, name: str, definition: Union[List[str], str, dict], + mapping: List[str] = []): + self.name = name + self.definition = definition + self.mapping = mapping + + def __str__(self): # pragma: no cover + return "\nSubscript range definition: %s\n\t%s\n" % ( + self.name, + "%s <- %s" % (self.definition, self.mapping) + if self.mapping else self.definition) + + @property + def _verbose(self) -> str: # pragma: no cover + """Get subscript range information.""" + return self.__str__() + + @property + def verbose(self): # pragma: no cover + """Print subscript range information to standard output.""" + print(self._verbose) + + def get_abstract_subscript_range(self) -> AbstractSubscriptRange: + """ + Instantiates an AbstractSubscriptRange object used for building. + This method is automatically called by the Sections's + get_abstract_section method. + + Returns + ------- + AbstractSubscriptRange: AbstractSubscriptRange + AbstractSubscriptRange object that can be used for building + the model in another programming language. + + """ + return AbstractSubscriptRange( + name=self.name, + subscripts=self.definition, + mapping=self.mapping + ) + + +class Component(): + """ + Model component defined by "name = expr" in Vensim. + + Parameters + ---------- + name: str + The original name of the component. + + subscripts: tuple + Tuple of length two with the list of subscripts + in the variable definition as first argument and the list of + subscripts that appears after the :EXCEPT: keyword (if used) as + the second argument. + + expression: str + The RHS of the element, expression to parse. + + """ + _kind = "Model component" + + def __init__(self, name: str, subscripts: Tuple[list, list], + expression: str): + self.name = name + self.subscripts = subscripts + self.expression = expression + + def __str__(self): # pragma: no cover + text = "\n%s definition: %s" % (self._kind, self.name) + text += "\nSubscrips: %s" % repr(self.subscripts[0])\ + if self.subscripts[0] else "" + text += " EXCEPT %s" % repr(self.subscripts[1])\ + if self.subscripts[1] else "" + text += "\n\t%s" % self._expression + return text + + @property + def _expression(self): # pragma: no cover + if hasattr(self, "ast"): + return str(self.ast).replace("\n", "\n\t") + + else: + return self.expression.replace("\n", "\n\t") + + @property + def _verbose(self) -> str: # pragma: no cover + """Get component information.""" + return self.__str__() + + @property + def verbose(self): # pragma: no cover + """Print component information to standard output.""" + print(self._verbose) + + def parse(self) -> None: + """ + Parse Component object with parsimonious using the grammar given + in 'parsing_grammars/components.peg' and the class EquationVisitor + to visit the RHS of the expressions. + + """ + tree = vu.Grammar.get("components", parsing_ops).parse(self.expression) + self.ast = EquationVisitor(tree).translation + + if isinstance(self.ast, structures["get_xls_lookups"]): + self.lookup = True + else: + self.lookup = False + + def get_abstract_component(self) -> Union[AbstractComponent, + AbstractLookup]: + """ + Get Abstract Component used for building. This method is + automatically called by Sections's get_abstract_section method. + + Returns + ------- + AbstractComponent: AbstractComponent or AbstractLookup + Abstract Component object that can be used for building + the model in another language. If the component equations + include external lookups (GET XLS/DIRECT LOOKUPS), an + AbstractLookup class will be used. + + """ + if self.lookup: + # get lookups equations + return AbstractLookup(subscripts=self.subscripts, ast=self.ast) + else: + return AbstractComponent(subscripts=self.subscripts, ast=self.ast) + + +class UnchangeableConstant(Component): + """ + Unchangeable constant defined by "name == expr" in Vensim. + This class inherits from the Component class. + + Parameters + ---------- + name: str + The original name of the component. + + subscripts: tuple + Tuple of length two with the list of subscripts + in the variable definition as first argument and the list of + subscripts that appears after the :EXCEPT: keyword (if used) as + second argument. + + expression: str + The RHS of the element, expression to parse. + + """ + _kind = "Unchangeable constant component" + + def __init__(self, name: str, subscripts: Tuple[list, list], + expression: str): + super().__init__(name, subscripts, expression) + + def get_abstract_component(self) -> AbstractUnchangeableConstant: + """ + Get Abstract Component used for building. This method is + automatically called by Sections's get_abstract_section method. + + Returns + ------- + AbstractComponent: AbstractUnchangeableConstant + Abstract Component object that can be used for building + the model in another language. + + """ + return AbstractUnchangeableConstant( + subscripts=self.subscripts, ast=self.ast) + + +class Lookup(Component): + """ + Lookup component, defined by "name(expr)" in Vensim. + This class inherits from the Component class. + + Parameters + ---------- + name: str + The original name of the component. + + subscripts: tuple + Tuple of length two with the list of subscripts in the variable + definition as first argument and the list of subscripts that appear + after the :EXCEPT: keyword (if used) as second argument. + + expression: str + The RHS of the element, expression to parse. + + """ + _kind = "Lookup component" + + def __init__(self, name: str, subscripts: Tuple[list, list], + expression: str): + super().__init__(name, subscripts, expression) + + def parse(self) -> None: + """ + Parse component object with parsimonious using the grammar given + in 'parsing_grammars/lookups.peg' and the class LookupsVisitor + to visit the RHS of the expressions. + """ + tree = vu.Grammar.get("lookups").parse(self.expression) + self.ast = LookupsVisitor(tree).translation + + def get_abstract_component(self) -> AbstractLookup: + """ + Get Abstract Component used for building. This method is + automatically called by Sections's get_abstract_section method. + + Returns + ------- + AbstractComponent: AbstractLookup + Abstract Component object that may be used for building + the model in another language. + + """ + return AbstractLookup(subscripts=self.subscripts, ast=self.ast) + + +class Data(Component): + """ + Data component, defined by "name := expr" in Vensim. + This class inherits from the Component class. + + Parameters + ---------- + name: str + The original name of the component. + + subscripts: tuple + Tuple of length two with the list of subscripts in the variable + definition as first argument and the list of subscripts that appear + after the :EXCEPT: keyword (if used) as second argument. + + keyword: str + The keyword used before the ":=" symbol. The following values are + possible: 'interpolate', 'raw', 'hold_backward' and 'look_forward'. + + expression: str + The RHS of the element, expression to parse. + + """ + _kind = "Data component" + + def __init__(self, name: str, subscripts: Tuple[list, list], + keyword: str, expression: str): + super().__init__(name, subscripts, expression) + self.keyword = keyword + + def __str__(self): # pragma: no cover + text = "\n%s definition: %s" % (self._kind, self.name) + text += "\nSubscrips: %s" % repr(self.subscripts[0])\ + if self.subscripts[0] else "" + text += " EXCEPT %s" % repr(self.subscripts[1])\ + if self.subscripts[1] else "" + text += "\nKeyword: %s" % self.keyword if self.keyword else "" + text += "\n\t%s" % self._expression + return text + + def parse(self) -> None: + """ + Parse component object with parsimonious using the grammar given + in 'parsing_grammars/components.peg' and the class EquationVisitor + to visit the RHS of the expressions. + + If the expression is None, the data will be read from a VDF file in + Vensim. + + """ + if not self.expression: + # empty data vars, read from vdf file + self.ast = structures["data"]() + else: + super().parse() + + def get_abstract_component(self) -> AbstractData: + """ + Get Abstract Component used for building. This method is + automatically called by Sections's get_abstract_section method. + + Returns + ------- + AbstractComponent: AbstractData + Abstract Component object that can be used for building + the model in another language. + + """ + return AbstractData( + subscripts=self.subscripts, ast=self.ast, keyword=self.keyword) + + +class LookupsVisitor(parsimonious.NodeVisitor): + """Visit the elements of a lookups to get the AST""" + def __init__(self, ast): + self.translation = None + self.visit(ast) + + def visit_limits(self, n, vc): + return n.text.strip()[:-1].replace(")-(", "),(") + + def visit_regularLookup(self, n, vc): + if vc[0]: + xy_limits = np.array(eval(vc[0])) + else: + xy_limits = np.full((2, 2), np.nan) + + values = np.array((eval(vc[2]))) + values = values[np.argsort(values[:, 0])] + + self.translation = structures["lookup"]( + x=tuple(values[:, 0]), + y=tuple(values[:, 1]), + x_limits=tuple(xy_limits[:, 0]), + y_limits=tuple(xy_limits[:, 1]), + type="interpolate" + ) + + def visit_excelLookup(self, n, vc): + arglist = vc[3].split(",") + + self.translation = structures["get_xls_lookups"]( + file=eval(arglist[0]), + tab=eval(arglist[1]), + x_row_or_col=eval(arglist[2]), + cell=eval(arglist[3]) + ) + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text + + +class EquationVisitor(parsimonious.NodeVisitor): + """Visit the elements of a equation to get the AST""" + def __init__(self, ast): + self.translation = None + self.elements = {} + self.subs = None # the subscripts if given + self.negatives = set() + self.visit(ast) + + def visit_expr_type(self, n, vc): + self.translation = self.elements[vc[0]] + + def visit_final_expr(self, n, vc): + # expressions with logical binary operators (:AND:, :OR:) + return vu.split_arithmetic( + structures["logic"], parsing_ops["logic_ops"], + "".join(vc).strip(), self.elements) + + def visit_logic_expr(self, n, vc): + # expressions with logical unitary operators (:NOT:) + id = vc[2] + if vc[0].lower() == ":not:": + id = self.add_element(structures["logic"]( + [":NOT:"], + (self.elements[id],) + )) + return id + + def visit_comp_expr(self, n, vc): + # expressions with comparisons (=, <>, <, <=, >, >=) + return vu.split_arithmetic( + structures["logic"], parsing_ops["comp_ops"], + "".join(vc).strip(), self.elements) + + def visit_add_expr(self, n, vc): + # expressions with additions (+, -) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["add_ops"], + "".join(vc).strip(), self.elements) + + def visit_prod_expr(self, n, vc): + # expressions with products (*, /) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["prod_ops"], + "".join(vc).strip(), self.elements) + + def visit_exp_expr(self, n, vc): + # expressions with exponentials (^) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["exp_ops"], + "".join(vc).strip(), self.elements, self.negatives) + + def visit_neg_expr(self, n, vc): + id = vc[2] + if vc[0] == "-": + if isinstance(self.elements[id], (float, int)): + self.elements[id] = -self.elements[id] + else: + self.negatives.add(id) + return id + + def visit_call(self, n, vc): + func = self.elements[vc[0]] + args = self.elements[vc[4]] + if func.reference in structures: + return self.add_element(structures[func.reference](*args)) + else: + return self.add_element(structures["call"](func, args)) + + def visit_reference(self, n, vc): + id = self.add_element(structures["reference"]( + vc[0].lower().replace(" ", "_"), self.subs)) + self.subs = None + return id + + def visit_limits(self, n, vc): + return self.add_element(n.text.strip()[:-1].replace(")-(", "),(")) + + def visit_lookup_with_def(self, n, vc): + if vc[10]: + xy_limits = np.array(eval(self.elements[vc[10]])) + else: + xy_limits = np.full((2, 2), np.nan) + + values = np.array((eval(vc[11]))) + values = values[np.argsort(values[:, 0])] + + lookup = structures["lookup"]( + x=tuple(values[:, 0]), + y=tuple(values[:, 1]), + x_limits=tuple(xy_limits[:, 0]), + y_limits=tuple(xy_limits[:, 1]), + type="interpolate" + ) + + return self.add_element(structures["with_lookup"]( + self.elements[vc[4]], lookup)) + + def visit_array(self, n, vc): + if ";" in n.text or "," in n.text: + return self.add_element(np.squeeze(np.array( + [row.split(",") for row in n.text.strip(";").split(";")], + dtype=float))) + else: + return self.add_element(eval(n.text)) + + def visit_subscript_list(self, n, vc): + subs = [x.strip() for x in vc[2].split(",")] + self.subs = structures["subscripts_ref"](subs) + return "" + + def visit_name(self, n, vc): + return n.text.strip() + + def visit_expr(self, n, vc): + if vc[0] not in self.elements: + return self.add_element(eval(vc[0])) + else: + return vc[0] + + def visit_string(self, n, vc): + return self.add_element(eval(n.text)) + + def visit_arguments(self, n, vc): + arglist = tuple(x.strip(",") for x in vc) + return self.add_element(tuple( + self.elements[arg] if arg in self.elements + else eval(arg) for arg in arglist)) + + def visit_parens(self, n, vc): + return vc[2] + + def visit__(self, n, vc): + # handles whitespace characters + return "" + + def visit_nan(self, n, vc): + return self.add_element(np.nan) + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text + + def add_element(self, element): + return vu.add_element(self.elements, element) diff --git a/pysd/translators/vensim/vensim_file.py b/pysd/translators/vensim/vensim_file.py new file mode 100644 index 00000000..84488a89 --- /dev/null +++ b/pysd/translators/vensim/vensim_file.py @@ -0,0 +1,362 @@ +""" +The VensimFile class allows reading the original Vensim model file, +parsing it into Section elements using the FileSectionsVisitor, +parsing its sketch using SketchVisitor in order to classify the varibales +per view. The final result can be exported to an AbstractModel class in +order to build the model in another programming language. +""" +import re +from typing import Union, List +from pathlib import Path +import warnings +import parsimonious +from collections.abc import Mapping + +from ..structures.abstract_model import AbstractModel + +from . import vensim_utils as vu +from .vensim_section import Section +from .vensim_utils import supported_extensions + + +class VensimFile(): + """ + The VensimFile class allows parsing an mdl file. + When the object is created, the model file is automatically opened; + unnecessary tabs, whitespaces, and linebreaks are removed; and + the sketch is split from the model equations. + + Parameters + ---------- + mdl_path: str or pathlib.Path + Path to the Vensim model. + + encoding: str or None (optional) + Encoding of the source model file. If None, the encoding will be + read from the model, if the encoding is not defined in the model + file it will be set to 'UTF-8'. Default is None. + + """ + def __init__(self, mdl_path: Union[str, Path], + encoding: Union[None, str] = None): + self.mdl_path = Path(mdl_path) + self.root_path = self.mdl_path.parent + self.model_text = self._read(encoding) + self.sketch = "" + self.view_elements = None + self._split_sketch() + + def __str__(self): # pragma: no cover + return "\nVensim model file, loaded from:\n\t%s\n" % self.mdl_path + + @property + def _verbose(self) -> str: # pragma: no cover + """Get model information.""" + text = self.__str__() + for section in self.sections: + text += section._verbose + + return text + + @property + def verbose(self): # pragma: no cover + """Print model information to standard output.""" + print(self._verbose) + + def _read(self, encoding: Union[None, str]) -> str: + """ + Read a Vensim file and assign its content to self.model_text + + Returns + ------- + str: model file content + + """ + # check for model extension + if self.mdl_path.suffix.lower() not in supported_extensions: + raise ValueError( + "The file to translate, '%s' " % self.mdl_path + + "is not a Vensim model. It must end with .mdl extension." + ) + + if encoding is None: + # Try detecting the encoding from the file + encoding = vu._detect_encoding_from_file(self.mdl_path) + + with self.mdl_path.open("r", encoding=encoding, + errors="ignore") as in_file: + model_text = in_file.read() + + return model_text + + def _split_sketch(self) -> None: + """Split model from the sketch.""" + try: + split_model = self.model_text.split("\\\\\\---///", 1) + self.model_text = self._clean(split_model[0]) + # remove plots section, if it exists + self.sketch = split_model[1].split("///---\\\\\\")[0] + except LookupError: + pass + + def _clean(self, text: str) -> str: + """Remove unnecessary characters.""" + return re.sub(r"[\n\t\s]+", " ", re.sub(r"\\\n\t", " ", text)) + + def parse(self, parse_all: bool = True) -> None: + """ + Parse model file with parsimonious using the grammar given in + 'parsing_grammars/file_sections.peg' and the class FileSectionsVisitor + to visit the parsed expressions. + + This breaks the model file in VensimSections, which correspond to the + main model section and the macros. + + Parameters + ---------- + parse_all: bool (optional) + If True, the VensimSection objects created will be + automatically parsed. Otherwise, these objects will only be + added to self.sections but not parsed. Default is True. + + """ + # get model sections (__main__ + macros) + tree = vu.Grammar.get("file_sections").parse(self.model_text) + self.sections = FileSectionsVisitor(tree).entries + + # main section path (Python model file) + self.sections[0].path = self.mdl_path.with_suffix(".py") + + for section in self.sections[1:]: + # macrots paths + section.path = self.mdl_path.parent.joinpath( + self._clean_file_names(section.name)[0] + ).with_suffix(".py") + + if parse_all: + # parse all sections + for section in self.sections: + # parse each section + section.parse() + + def parse_sketch(self, subview_sep: List[str]) -> None: + """ + Parse the sketch of the model with parsimonious using the grammar + given in 'parsing_grammars/sketch.peg' and the class SketchVisitor + to visit the parsed expressions. + + It will modify the views_dict of the first section, including + the dictionary of the classification of variables by views. This, + method should be called after calling the self.parse method. + + Parameters + ---------- + subview_sep: list + List of the separators to use to classify the model views in + folders and subfolders. The sepparator must be ordered in the + same order they appear in the view name. For example, + if a view is named "economy:demand.exports" if + subview_sep=[":", "."] this view's variables will be included + in the file 'exports.py' and inside the folders economy/demand. + + + """ + if self.sketch: + sketch = list(map( + lambda x: x.strip(), + self.sketch.split("\\\\\\---/// ") + )) + else: + warnings.warn( + "No sketch detected. The model will be built in a " + "single file.") + return None + + grammar = vu.Grammar.get("sketch") + view_elements = {} + for module in sketch: + for sketch_line in module.split("\n"): + # parsed line could have information about new view name + # or of a variable inside a view + parsed = SketchVisitor(grammar.parse(sketch_line)) + + if parsed.view_name: + view_name = parsed.view_name + view_elements[view_name] = set() + + elif parsed.variable_name: + view_elements[view_name].add(parsed.variable_name) + + # removes views that do not include any variable in them + non_empty_views = { + key: value for key, value in view_elements.items() if value + } + + # split into subviews, if subview_sep is provided + views_dict = {} + + if len(non_empty_views) == 1: + warnings.warn( + "Only a single view with no subviews was detected. The model" + " will be built in a single file.") + return + elif subview_sep and any( + sep in view for sep in subview_sep for view in non_empty_views): + escaped_separators = list(map(lambda x: re.escape(x), subview_sep)) + for full_name, values in non_empty_views.items(): + # split the full view name using the separator and make the + # individual parts safe file or directory names + clean_view_parts = self._clean_file_names( + *re.split("|".join(escaped_separators), full_name)) + # creating a nested dict for each view.subview + # (e.g. {view_name: {subview_name: [values]}}) + nested_dict = values + + for item in reversed(clean_view_parts): + nested_dict = {item: nested_dict} + # merging the new nested_dict into the views_dict, preserving + # repeated keys + self._merge_nested_dicts(views_dict, nested_dict) + else: + # view names do not have separators or separator characters + # not provided + + if subview_sep and not any( + sep in view for sep in subview_sep for view in non_empty_views): + warnings.warn( + "The given subview separators were not matched in " + "any view name.") + + for view_name, elements in non_empty_views.items(): + views_dict[self._clean_file_names(view_name)[0]] = elements + + self.sections[0].split = True + self.sections[0].views_dict = views_dict + + def get_abstract_model(self) -> AbstractModel: + """ + Instantiate the AbstractModel object used during building. This, + method should be called after parsing the model (self.parse) and, + in case you want to split the variables per views, also after + parsing the sketch (self.parse_sketch). This automatically calls the + get_abstract_section method from the model sections. + + Returns + ------- + AbstractModel: AbstractModel + Abstract Model object that can be used for building the model + in another language. + + """ + return AbstractModel( + original_path=self.mdl_path, + sections=tuple(section.get_abstract_section() + for section in self.sections)) + + @staticmethod + def _clean_file_names(*args): + """ + Removes special characters and makes clean file names. + + Parameters + ---------- + *args: tuple + Any number of strings to clean. + + Returns + ------- + clean: list + List containing the clean strings. + + """ + return [ + re.sub( + r"[\W]+", "", + name.replace(" ", "_") + ).lstrip("0123456789") + for name in args] + + def _merge_nested_dicts(self, original_dict, dict_to_merge): + """ + Merge dictionaries recursively, preserving common keys. + + Parameters + ---------- + original_dict: dict + Dictionary onto which the merge is executed. + + dict_to_merge: dict + Dictionary to be merged to the original_dict. + + Returns + ------- + None + + """ + for key, value in dict_to_merge.items(): + if (key in original_dict and isinstance(original_dict[key], dict) + and isinstance(value, Mapping)): + self._merge_nested_dicts(original_dict[key], value) + else: + original_dict[key] = value + + +class FileSectionsVisitor(parsimonious.NodeVisitor): + """Parse file sections""" + def __init__(self, ast): + self.entries = [None] + self.visit(ast) + + def visit_main(self, n, vc): + # main will always be stored as the first entry + if self.entries[0] is None: + self.entries[0] = Section( + name="__main__", + path=Path("."), + section_type="main", + params=[], + returns=[], + content=n.text.strip(), + split=False, + views_dict=None + ) + else: + # this is needed when macro parts are in the middle of the file + self.entries[0].content += n.text.strip() + + def visit_macro(self, n, vc): + self.entries.append( + Section( + name=vc[2].strip().lower().replace(" ", "_"), + path=Path("."), + section_type="macro", + params=[ + x.strip() for x in vc[6].split(",")] if vc[6] else [], + returns=[ + x.strip() for x in vc[10].split(",")] if vc[10] else [], + content=vc[13].strip(), + split=False, + views_dict=None + ) + ) + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text or "" + + +class SketchVisitor(parsimonious.NodeVisitor): + """Sketch visitor to save the view names and the variables in each""" + def __init__(self, ast): + self.variable_name = None + self.view_name = None + self.visit(ast) + + def visit_view_name(self, n, vc): + self.view_name = n.text.lower() + + def visit_var_definition(self, n, vc): + if int(vc[10]) % 2 != 0: # not a shadow variable + self.variable_name = vc[4].replace(" ", "_").lower() + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text.strip() or "" diff --git a/pysd/translators/vensim/vensim_section.py b/pysd/translators/vensim/vensim_section.py new file mode 100644 index 00000000..e2f100bf --- /dev/null +++ b/pysd/translators/vensim/vensim_section.py @@ -0,0 +1,208 @@ +""" +The Section class allows parsing a model section into Elements using the +SectionElementsVisitor. The final result can be exported to an +AbstractSection class in order to build a model in another programming +language. +A section is either the main model (without the macros), or a +macro definition. +""" +from typing import List, Union +from pathlib import Path +import parsimonious + +from ..structures.abstract_model import AbstractElement, AbstractSection + +from . import vensim_utils as vu +from .vensim_element import Element, SubscriptRange, Component + + +class Section(): + """ + The Section class allows parsing the elements of a model section. + + Parameters + ---------- + name: str + Section name. That is, '__main__' for the main section, and the + macro name for macros. + + path: pathlib.Path + Section path. The model name for the main section and the clean + macro name for a macro. + + section_type: str ('main' or 'macro') + The section type. + + params: list + List of parameters that the section takes. If it is the main + section, it will be an empty list. + + returns: list + List of variables that returns the section. If it is the main + section, it will be an empty list. + + content: str + Section content as string. + + split: bool + If True, the created section will split the variables + depending on the views_dict. + + views_dict: dict + The dictionary of the views. Giving the variables classified at + any level in order to split them by files. + + """ + def __init__(self, name: str, path: Path, section_type: str, + params: List[str], returns: List[str], + content: str, split: bool, views_dict: Union[dict, None]): + self.name = name + self.path = path + self.type = section_type + self.params = params + self.returns = returns + self.content = content + self.split = split + self.views_dict = views_dict + self.elements = None + + def __str__(self): # pragma: no cover + return "\nSection: %s\n" % self.name + + @property + def _verbose(self) -> str: # pragma: no cover + """Get section information.""" + text = self.__str__() + if self.elements: + for element in self.elements: + text += element._verbose + else: + text += self.content + + return text + + @property + def verbose(self): # pragma: no cover + """Print section information to standard output.""" + print(self._verbose) + + def parse(self, parse_all: bool = True) -> None: + """ + Parse section object with parsimonious using the grammar given in + 'parsing_grammars/section_elements.peg' and the class + SectionElementsVisitor to visit the parsed expressions. + + This will break the section (__main__ or macro) in VensimElements, + which are each model expression LHS and RHS with already parsed + units and description. + + Parameters + ---------- + parse_all: bool (optional) + If True, the created VensimElement objects will be + automatically parsed. Otherwise, these objects will only be + added to self.elements but not parsed. Default is True. + + """ + # parse the section to get the elements + tree = vu.Grammar.get("section_elements").parse(self.content) + self.elements = SectionElementsParser(tree).entries + + if parse_all: + # parse all elements + self.elements = [element.parse() for element in self.elements] + + # split subscript from other components + self.subscripts = [ + element for element in self.elements + if isinstance(element, SubscriptRange) + ] + self.components = [ + element for element in self.elements + if isinstance(element, Component) + ] + + # reorder element list for better printing + self.elements = self.subscripts + self.components + + [component.parse() for component in self.components] + + def get_abstract_section(self) -> AbstractSection: + """ + Instantiate an object of the AbstractSection class that will be used + during the building process. This method should be called after + parsing the section (self.parse). This method is automatically called + by Model's get_abstract_model method, and automatically generates the + AbstractSubscript ranges and merges the components in elements. It also + calls the get_abstract_components method from each model component. + + Returns + ------- + AbstractSection: AbstractSection + AbstractSection object that can be used for building the model + in another programming language. + + """ + return AbstractSection( + name=self.name, + path=self.path, + type=self.type, + params=self.params, + returns=self.returns, + subscripts=[ + subs_range.get_abstract_subscript_range() + for subs_range in self.subscripts + ], + elements=self._merge_components(), + split=self.split, + views_dict=self.views_dict + ) + + def _merge_components(self) -> List[AbstractElement]: + """Merge model components by their name.""" + merged = {} + for component in self.components: + # get a safe name to merge (case and white/underscore sensitivity) + name = component.name.lower().replace(" ", "_") + if name not in merged: + # create new element if it is the first component + merged[name] = AbstractElement( + name=component.name, + components=[]) + + if component.units: + # add units to element data + merged[name].units = component.units + if component.limits: + # add limits to element data + merged[name].limits = component.limits + if component.documentation: + # add documentation to element data + merged[name].documentation = component.documentation + + # add AbstractComponent to the list of components + merged[name].components.append(component.get_abstract_component()) + + return list(merged.values()) + + +class SectionElementsParser(parsimonious.NodeVisitor): + """ + Visit section elements to get their equation, units and documentation. + """ + # TODO include units parsing + def __init__(self, ast): + self.entries = [] + self.visit(ast) + + def visit_entry(self, n, vc): + self.entries.append( + Element( + equation=vc[0].strip(), + units=vc[2].strip(), + documentation=vc[4].strip(), + ) + ) + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text or "" diff --git a/pysd/translators/vensim/vensim_structures.py b/pysd/translators/vensim/vensim_structures.py new file mode 100644 index 00000000..84e1f5a8 --- /dev/null +++ b/pysd/translators/vensim/vensim_structures.py @@ -0,0 +1,61 @@ +""" +The AST structures are created with the help of the parsimonious visitors +using the structures dictionary. + +""" +import re +from ..structures import abstract_expressions as ae + + +structures = { + "reference": ae.ReferenceStructure, + "subscripts_ref": ae.SubscriptsReferenceStructure, + "arithmetic": ae.ArithmeticStructure, + "logic": ae.LogicStructure, + "with_lookup": ae.InlineLookupsStructure, + "call": ae.CallStructure, + "game": ae.GameStructure, + "get_xls_lookups": ae.GetLookupsStructure, + "get_direct_lookups": ae.GetLookupsStructure, + "get_xls_data": ae.GetDataStructure, + "get_direct_data": ae.GetDataStructure, + "get_xls_constants": ae.GetConstantsStructure, + "get_direct_constants": ae.GetConstantsStructure, + "initial": ae.InitialStructure, + "integ": ae.IntegStructure, + "delay1": lambda x, y: ae.DelayStructure(x, y, x, 1), + "delay1i": lambda x, y, z: ae.DelayStructure(x, y, z, 1), + "delay3": lambda x, y: ae.DelayStructure(x, y, x, 3), + "delay3i": lambda x, y, z: ae.DelayStructure(x, y, z, 3), + "delay_n": ae.DelayNStructure, + "delay_fixed": ae.DelayFixedStructure, + "smooth": lambda x, y: ae.SmoothStructure(x, y, x, 1), + "smoothi": lambda x, y, z: ae.SmoothStructure(x, y, z, 1), + "smooth3": lambda x, y: ae.SmoothStructure(x, y, x, 3), + "smooth3i": lambda x, y, z: ae.SmoothStructure(x, y, z, 3), + "smooth_n": ae.SmoothNStructure, + "trend": ae.TrendStructure, + "forecast": lambda x, y, z: ae.ForecastStructure(x, y, z, 0), + "sample_if_true": ae.SampleIfTrueStructure, + "lookup": ae.LookupsStructure, + "data": ae.DataStructure, + "pulse_train": lambda start, width, interval, end: ae.CallStructure( + ae.ReferenceStructure("pulse_train"), + (start, interval, width, end)) +} + +operators = { + "logic_ops": [":AND:", ":OR:"], + "not_ops": [":NOT:"], + "comp_ops": ["=", "<>", "<=", "<", ">=", ">"], + "add_ops": ["+", "-"], + "prod_ops": ["*", "/"], + "exp_ops": ["^"], + "pre_ops": ["+", "-"] +} + + +parsing_ops = { + key: "|".join(re.escape(x) for x in values) + for key, values in operators.items() +} diff --git a/pysd/translators/vensim/vensim_utils.py b/pysd/translators/vensim/vensim_utils.py new file mode 100644 index 00000000..ea01bb0d --- /dev/null +++ b/pysd/translators/vensim/vensim_utils.py @@ -0,0 +1,155 @@ +import re +import warnings +import uuid + +import parsimonious +from typing import Dict +from pathlib import Path +from chardet import detect + + +supported_extensions = [".mdl"] + + +class Grammar(): + _common_grammar = None + _grammar_path: Path = Path(__file__).parent.joinpath("parsing_grammars") + _grammar: Dict = {} + + @classmethod + def get(cls, grammar: str, subs: dict = {}) -> parsimonious.Grammar: + """Get parsimonious grammar for parsing""" + if grammar not in cls._grammar: + # include grammar in the class singleton + cls._grammar[grammar] = parsimonious.Grammar( + cls._read_grammar(grammar) % subs + ) + + return cls._grammar[grammar] + + @classmethod + def _read_grammar(cls, grammar: str) -> str: + """Read grammar from a file and include common grammar""" + with cls._gpath(grammar).open(encoding="ascii") as gfile: + source_grammar: str = gfile.read() + + return cls._include_common_grammar(source_grammar) + + @classmethod + def _include_common_grammar(cls, source_grammar: str) -> str: + """Include common grammar""" + if not cls._common_grammar: + with cls._gpath("common_grammar").open(encoding="ascii") as gfile: + cls._common_grammar: str = gfile.read() + + return r"{source_grammar}{common_grammar}".format( + source_grammar=source_grammar, common_grammar=cls._common_grammar + ) + + @classmethod + def _gpath(cls, grammar: str) -> Path: + """Get the grammar file path""" + return cls._grammar_path.joinpath(grammar).with_suffix(".peg") + + +def split_arithmetic(structure: object, parsing_ops: dict, + expression: str, elements: dict, + negatives: set = set()) -> object: + """ + Split arithmetic pattern and return the corresponding object. + + Parameters + ---------- + structure: callable + Callable that generates the arithmetic object to return. + parsing_ops: dict + The parsing operators dictionary. + expression: str + Original expression with the operator and the hex code to the objects. + elements: dict + Dictionary of the hex identifiers and the objects that represent. + negative: set + Set of element hex values that must change their sign. + + Returns + ------- + object: structure + Final object of the arithmetic operation or initial object if + no operations are performed. + + """ + pattern = re.compile(parsing_ops) + parts = pattern.split(expression) # list of elements ids + ops = pattern.findall(expression) # operators list + if not ops: + # no operators return original object + if parts[0] in negatives: + # make original object negative + negatives.remove(parts[0]) + return add_element( + elements, + structure(["negative"], (elements[parts[0]],))) + else: + return expression + else: + if not negatives: + # create arithmetic object + return add_element( + elements, + structure( + ops, + tuple([elements[id] for id in parts]))) + else: + # manage negative expressions + current_id = parts.pop() + current = elements[current_id] + if current_id in negatives: + negatives.remove(current_id) + current = structure(["negative"], (current,)) + while ops: + current_id = parts.pop() + current = structure( + [ops.pop()], + (elements[current_id], current)) + if current_id in negatives: + negatives.remove(current_id) + current = structure(["negative"], (current,)) + + return add_element(elements, current) + + +def add_element(elements: dict, element: object) -> str: + """ + Add element to elements dict using an unique hex identifier + + Parameters + ---------- + elements: dict + Dictionary of all elements. + + element: object + Element to add. + + Returns + ------- + id: str (hex) + The name of the key where element is saved in elements. + + """ + id = uuid.uuid4().hex + elements[id] = element + return id + + +def _detect_encoding_from_file(mdl_file: Path) -> str: + """Detect and return the encoding from a Vensim file""" + try: + with mdl_file.open("rb") as in_file: + f_line: bytes = in_file.readline() + f_line: str = f_line.decode(detect(f_line)['encoding']) + return re.search(r"(?<={)(.*)(?=})", f_line).group() + except (AttributeError, UnicodeDecodeError): + warnings.warn( + "No encoding specified or detected to translate the model " + "file. 'UTF-8' encoding will be used.") + return "UTF-8" diff --git a/pysd/translators/xmile/__init__.py b/pysd/translators/xmile/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pysd/translators/xmile/parsing_grammars/equations.peg b/pysd/translators/xmile/parsing_grammars/equations.peg new file mode 100644 index 00000000..5a0a659a --- /dev/null +++ b/pysd/translators/xmile/parsing_grammars/equations.peg @@ -0,0 +1,51 @@ +# Parsing Expression Grammar: components + +expr_type = array / final_expr + +final_expr = conditional_statement / logic2_expr + +logic2_expr = logic_expr (_ logic_oper _ logic_expr)* # logic operators (:and:, :or:) +logic_expr = not_oper? _ comp_expr # :not: operator +comp_expr = add_expr (_ comp_oper _ add_expr)? # comparison (e.g. '<', '=>') +add_expr = mod_expr (_ add_oper _ mod_expr)* # addition and substraction +mod_expr = prod_expr (_ "mod" _ prod_expr)? # modulo +prod_expr = exp_expr (_ prod_oper _ exp_expr)* # product and division +exp_expr = neg_expr (_ exp_oper _ neg_expr)* # exponential +neg_expr = pre_oper? _ expr # pre operators (-, +) +expr = call / parens / number / reference + +arguments = (final_expr _ ","? _)* +parens = "(" _ final_expr _ ")" + +call = reference _ "(" _ arguments _ ")" +conditional_statement = "IF" _ logic2_expr _ "THEN" _ logic2_expr _ "ELSE" _ logic2_expr + +reference = (name _ subscript_list) / name # check first for subscript +subscript_list = "[" _ (name _ "!"? _ ","? _)+ _ "]" + +array = (raw_number _ ("," / ";")? _)+ !~r"." # negative lookahead for + +logic_oper = ~r"(%(logic_ops)s)"IU +not_oper = ~r"(%(not_ops)s)"IU +comp_oper = ~r"(%(comp_ops)s)"IU +add_oper = ~r"(%(add_ops)s)"IU +prod_oper = ~r"(%(prod_ops)s)"IU +exp_oper = ~r"(%(exp_ops)s)"IU +pre_oper = ~r"(%(pre_ops)s)"IU + +_ = spacechar* +spacechar = " "* ~"\t"* + +name = basic_id / escape_group + +# This takes care of models with Unicode variable names +basic_id = id_start id_continue* + +id_start = ~r"[\w]"IU +id_continue = id_start / ~r"[0-9\'\$\_]" + +# between quotes, either escaped quote or character that is not a quote +escape_group = "\"" ( "\\\"" / ~r"[^\"]" )* "\"" + +number = raw_number +raw_number = ("+"/"-")? (~r"\d+\.?\d*([eE][+-]?\d+)?" / ~r"\.\d+([eE][+-]?\d+)?") diff --git a/pysd/translators/xmile/xmile_element.py b/pysd/translators/xmile/xmile_element.py new file mode 100644 index 00000000..b9180678 --- /dev/null +++ b/pysd/translators/xmile/xmile_element.py @@ -0,0 +1,655 @@ +""" +The Element class child classes alow parsing the expressions of a +given model element. There are 3 tipes of elements: + +- Flows and auxiliars (Flaux class): Regular elements, defined with + or . +- Gfs (Gf class): Lookup elements, defined with . +- Stocks (Stock class): Data component, defined with + +Moreover, a 4 type element is defined ControlElement, which allows parsing +the values of the model control variables (time step, initialtime, final time). + +The final result from a parsed element can be exported to an +AbstractElement object in order to build a model in other language. +""" +import re +from typing import Tuple, Union, List +from lxml import etree +import parsimonious +import numpy as np + +from ..structures.abstract_model import\ + AbstractElement, AbstractLookup, AbstractComponent, AbstractSubscriptRange + +from ..structures.abstract_expressions import AbstractSyntax + +from . import xmile_utils as vu +from .xmile_structures import structures, parsing_ops + + +class Element(): + """ + Element class. This class provides the shared methods for its childs: + Flaux, Gf, Stock, and ControlElement. + + Parameters + ---------- + node: etree._Element + The element node content. + + ns: dict + The namespace of the section. + + subscripts: dict + The subscript dictionary of the section, necessary to parse + some subscripted elements. + + """ + _interp_methods = { + "continuous": "interpolate", + "extrapolate": "extrapolate", + "discrete": "hold_backward" + } + + _kind = "Element" + + def __init__(self, node: etree._Element, ns: dict, subscripts): + self.node = node + self.ns = ns + self.name = node.attrib["name"].replace("\\n", " ") + self.units = self._get_xpath_text(node, "ns:units") or "" + self.documentation = self._get_xpath_text(node, "ns:doc") or "" + self.limits = (None, None) + self.components = [] + self.subscripts = subscripts + + def __str__(self): # pragma: no cover + text = "\n%s definition: %s" % (self._kind, self.name) + text += "\nSubscrips: %s" % repr(self.subscripts)\ + if self.subscripts else "" + text += "\n\t%s" % self._expression + return text + + @property + def _expression(self): # pragma: no cover + if hasattr(self, "ast"): + return str(self.ast).replace("\n", "\n\t") + + else: + return self.node.text.replace("\n", "\n\t") + + @property + def _verbose(self) -> str: # pragma: no cover + """Get element information.""" + return self.__str__() + + @property + def verbose(self): # pragma: no cover + """Print element information to standard output.""" + print(self._verbose) + + def _get_xpath_text(self, node: etree._Element, + xpath: str) -> Union[str, None]: + """Safe access of occassionally missing text""" + try: + return node.xpath(xpath, namespaces=self.ns)[0].text + except IndexError: + return None + + def _get_xpath_attrib(self, node: etree._Element, + xpath: str, attrib: str) -> Union[str, None]: + """Safe access of occassionally missing attributes""" + # defined here to take advantage of NS in default + try: + return node.xpath(xpath, namespaces=self.ns)[0].attrib[attrib] + except IndexError: + return None + + def _get_limits(self) -> Tuple[Union[None, str], Union[None, str]]: + """Get the limits of the element""" + lims = ( + self._get_xpath_attrib(self.node, 'ns:range', 'min'), + self._get_xpath_attrib(self.node, 'ns:range', 'max') + ) + return tuple(float(x) if x is not None else x for x in lims) + + def _parse_lookup_xml_node(self, node: etree._Element) -> AbstractSyntax: + """ + Parse lookup definition + + Returns + ------- + AST: AbstractSyntax + + """ + ys_node = node.xpath('ns:ypts', namespaces=self.ns)[0] + ys = np.fromstring( + ys_node.text, + dtype=float, + sep=ys_node.attrib['sep'] if 'sep' in ys_node.attrib else ',' + ) + xscale_node = node.xpath('ns:xscale', namespaces=self.ns) + if len(xscale_node) > 0: + xmin = xscale_node[0].attrib['min'] + xmax = xscale_node[0].attrib['max'] + xs = np.linspace(float(xmin), float(xmax), len(ys)) + else: + xs_node = node.xpath('ns:xpts', namespaces=self.ns)[0] + xs = np.fromstring( + xs_node.text, + dtype=float, + sep=xs_node.attrib['sep'] if 'sep' in xs_node.attrib else ',' + ) + + interp = node.attrib['type'] if 'type' in node.attrib else 'continuous' + + return structures["lookup"]( + x=tuple(xs[np.argsort(xs)]), + y=tuple(ys[np.argsort(xs)]), + x_limits=(np.min(xs), np.max(xs)), + y_limits=(np.min(ys), np.max(ys)), + type=self._interp_methods[interp] + ) + + def parse(self) -> None: + """Parse all the components of an element""" + if self.node.xpath("ns:element", namespaces=self.ns): + # defined in several equations each with one subscript + for subnode in self.node.xpath("ns:element", namespaces=self.ns): + self.components.append( + ((subnode.attrib["subscript"].split(","), []), + self._parse_component(subnode)[0]) + ) + else: + # get the subscripts from element + subscripts = [ + subnode.attrib["name"] + for subnode + in self.node.xpath("ns:dimensions/ns:dim", namespaces=self.ns) + ] + parsed = self._parse_component(self.node) + if len(parsed) == 1: + # element defined with one equation + self.components = [((subscripts, []), parsed[0])] + else: + # element defined in several equations, but only the general + # subscripts are given, save each equation with its + # subscrtipts + subs_list = self.subscripts[subscripts[0]] + self.components = [ + (([subs], []), parsed_i) for subs, parsed_i in + zip(subs_list, parsed) + ] + + def _smile_parser(self, expression: str) -> AbstractSyntax: + """ + Parse expression with parsimonious. + + Returns + ------- + AST: AbstractSyntax + + """ + tree = vu.Grammar.get("equations", parsing_ops).parse( + expression.strip()) + return EquationVisitor(tree).translation + + def _get_empty_abstract_element(self) -> AbstractElement: + """ + Get empty Abstract used for building + + Returns + ------- + AbstractElement + """ + return AbstractElement( + name=self.name, + units=self.units, + limits=self.limits, + documentation=self.documentation, + components=[]) + + +class Flaux(Element): + """ + Flow or auxiliary variable definde by or in Xmile. + + Parameters + ---------- + node: etree._Element + The element node content. + + ns: dict + The namespace of the section. + + subscripts: dict + The subscript dictionary of the section, necessary to parse + some subscripted elements. + + """ + _kind = "Flaux" + + def __init__(self, node, ns, subscripts): + super().__init__(node, ns, subscripts) + self.limits = self._get_limits() + + def _parse_component(self, node: etree._Element) -> List[AbstractSyntax]: + """ + Parse one Flaux component + + Returns + ------- + AST: AbstractSyntax + + """ + asts = [] + for eqn in node.xpath('ns:eqn', namespaces=self.ns): + # Replace new lines with space, and replace 2 or more spaces with + # single space. Then ensure there is no space at start or end of + # equation + eqn = re.sub(r"(\s{2,})", " ", eqn.text.replace("\n", ' ')).strip() + ast = self._smile_parser(eqn) + + gf_node = self.node.xpath("ns:gf", namespaces=self.ns) + if len(gf_node) > 0: + ast = structures["inline_lookup"]( + ast, self._parse_lookup_xml_node(gf_node[0])) + asts.append(ast) + + return asts + + def get_abstract_element(self) -> AbstractElement: + """ + Get Abstract Element used for building. This method is + automatically called by Sections's get_abstract_section. + + Returns + ------- + AbstractElement: AbstractElement + Abstract Element object that can be used for building + the model in another language. It contains a list of + AbstractComponents with the Abstract Syntax Tree of each of + the expressions. + + """ + ae = self._get_empty_abstract_element() + for component in self.components: + ae.components.append(AbstractComponent( + subscripts=component[0], + ast=component[1])) + return ae + + +class Gf(Element): + """ + Gf variable (lookup) definde by in Xmile. + + Parameters + ---------- + node: etree._Element + The element node content. + + ns: dict + The namespace of the section. + + subscripts: dict + The subscript dictionary of the section, necessary to parse + some subscripted elements. + + """ + _kind = "Gf component" + + def __init__(self, node, ns, subscripts): + super().__init__(node, ns, subscripts) + self.limits = self.get_limits() + + def get_limits(self) -> Tuple[Union[None, str], Union[None, str]]: + """Get the limits of the Gf element""" + lims = ( + self._get_xpath_attrib(self.node, 'ns:yscale', 'min'), + self._get_xpath_attrib(self.node, 'ns:yscale', 'max') + ) + return tuple(float(x) if x is not None else x for x in lims) + + def _parse_component(self, node: etree._Element) -> AbstractSyntax: + """ + Parse one Gf component + + Returns + ------- + AST: AbstractSyntax + + """ + return [self._parse_lookup_xml_node(self.node)] + + def get_abstract_element(self) -> AbstractElement: + """ + Get Abstract Element used for building. This method is + automatically called by Sections's get_abstract_section. + + Returns + ------- + AbstractElement: AbstractElement + Abstract Element object that can be used for building + the model in another language. It contains a list of + AbstractComponents with the Abstract Syntax Tree of each of + the expressions. + + """ + ae = self._get_empty_abstract_element() + for component in self.components: + ae.components.append(AbstractLookup( + subscripts=component[0], + ast=component[1])) + return ae + + +class Stock(Element): + """ + Stock variable definde by in Xmile. + + Parameters + ---------- + node: etree._Element + The element node content. + + ns: dict + The namespace of the section. + + subscripts: dict + The subscript dictionary of the section, necessary to parse + some subscripted elements. + + """ + + _kind = "Stock component" + + def __init__(self, node, ns, subscripts): + super().__init__(node, ns, subscripts) + self.limits = self._get_limits() + + def _parse_component(self, node) -> AbstractSyntax: + """ + Parse one Stock component + + Returns + ------- + AST: AbstractSyntax + + """ + # Parse each flow equations + inflows = [ + self._smile_parser(inflow.text) + for inflow in self.node.xpath('ns:inflow', namespaces=self.ns)] + outflows = [ + self._smile_parser(outflow.text) + for outflow in self.node.xpath('ns:outflow', namespaces=self.ns)] + + if inflows: + # stock has inflows + expr = ["+"] * (len(inflows)-1) + ["-"] * len(outflows) + elif outflows: + # stock has no inflows but outflows + outflows[0] = structures["negative"](outflows[0]) + expr = ["-"] * (len(outflows)-1) + else: + # stock is constant + expr = [] + inflows = [0] + + if expr: + # stock has more than one flow + flows = structures["arithmetic"](expr, inflows+outflows) + else: + # stock has only one flow + flows = inflows[0] if inflows else outflows[0] + + # Read the initial value equation for stock element + initial = self._smile_parser(self._get_xpath_text(self.node, 'ns:eqn')) + + return [structures["stock"](flows, initial)] + + def get_abstract_element(self) -> AbstractElement: + """ + Get Abstract Element used for building. This method is + automatically called by Sections's get_abstract_section. + + Returns + ------- + AbstractElement: AbstractElement + Abstract Element object that can be used for building + the model in another language. It contains a list of + AbstractComponents with the Abstract Syntax Tree of each of + the expressions. + + """ + ae = self._get_empty_abstract_element() + for component in self.components: + ae.components.append(AbstractComponent( + subscripts=component[0], + ast=component[1])) + return ae + + +class ControlElement(Element): + """Control variable (lookup)""" + _kind = "Control variable" + + def __init__(self, name, units, documentation, eqn): + self.name = name + self.units = units + self.documentation = documentation + self.limits = (None, None) + self.eqn = eqn + + def parse(self) -> None: + """ + Parse control elment. + + Returns + ------- + AST: AbstractSyntax + + """ + self.ast = self._smile_parser(self.eqn) + + def get_abstract_element(self) -> AbstractElement: + """ + Get Abstract Element used for building. This method is + automatically called by Sections's get_abstract_section. + + Returns + ------- + AbstractElement: AbstractElement + Abstract Element object that can be used for building + the model in another language. It contains an AbstractComponent + with the Abstract Syntax Tree of the expression. + + """ + ae = self._get_empty_abstract_element() + ae.components.append(AbstractComponent( + subscripts=([], []), + ast=self.ast)) + return ae + + +class SubscriptRange(): + """Subscript range definition.""" + + def __init__(self, name: str, definition: List[str], + mapping: List[str] = []): + self.name = name + self.definition = definition + self.mapping = mapping + + def __str__(self): # pragma: no cover + return "\nSubscript range definition: %s\n\t%s\n" % ( + self.name, + self.definition) + + @property + def _verbose(self) -> str: # pragma: no cover + """Get subscript range information.""" + return self.__str__() + + @property + def verbose(self): # pragma: no cover + """Print subscript range information to standard output.""" + print(self._verbose) + + def get_abstract_subscript_range(self) -> AbstractSubscriptRange: + """ + Get Abstract Subscript Range used for building. This method is + automatically called by Sections's get_abstract_section. + + Returns + ------- + AbstractSubscriptRange: AbstractSubscriptRange + Abstract Subscript Range object that can be used for building + the model in another language. + + """ + return AbstractSubscriptRange( + name=self.name, + subscripts=self.definition, + mapping=self.mapping + ) + + +class EquationVisitor(parsimonious.NodeVisitor): + """Visit the elements of a equation to get the AST""" + def __init__(self, ast): + self.translation = None + self.elements = {} + self.subs = None # the subscripts if given + self.negatives = set() + self.visit(ast) + + def visit_expr_type(self, n, vc): + self.translation = self.elements[vc[0]] + + def visit_logic2_expr(self, n, vc): + # expressions with logical binary operators (and, or) + return vu.split_arithmetic( + structures["logic"], parsing_ops["logic_ops"], + "".join(vc).strip(), self.elements) + + def visit_logic_expr(self, n, vc): + # expressions with logical unitary operators (not) + id = vc[2] + if vc[0].lower() == "not": + id = self.add_element(structures["logic"]( + [":NOT:"], + (self.elements[id],) + )) + return id + + def visit_comp_expr(self, n, vc): + # expressions with comparisons (=, <>, <, <=, >, >=) + return vu.split_arithmetic( + structures["logic"], parsing_ops["comp_ops"], + "".join(vc).strip(), self.elements) + + def visit_add_expr(self, n, vc): + # expressions with additions (+, -) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["add_ops"], + "".join(vc).strip(), self.elements) + + def visit_mod_expr(self, n, vc): + # modulo expressions (mod) + if vc[1].lower().startswith("mod"): + return self.add_element( + structures["call"]( + structures["reference"]("modulo"), + (self.elements[vc[0]], self.elements[vc[1][3:]]) + )) + else: + return vc[0] + + def visit_prod_expr(self, n, vc): + # expressions with products (*, /) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["prod_ops"], + "".join(vc).strip(), self.elements) + + def visit_exp_expr(self, n, vc): + # expressions with exponentials (^) + return vu.split_arithmetic( + structures["arithmetic"], parsing_ops["exp_ops"], + "".join(vc).strip(), self.elements, self.negatives) + + def visit_neg_expr(self, n, vc): + id = vc[2] + if vc[0] == "-": + if isinstance(self.elements[id], (float, int)): + self.elements[id] = -self.elements[id] + else: + self.negatives.add(id) + return id + + def visit_call(self, n, vc): + func = self.elements[vc[0]] + args = self.elements[vc[4]] + if func.reference in structures: + func_str = structures[func.reference] + if isinstance(func_str, dict): + return self.add_element(func_str[len(args)](*args)) + else: + return self.add_element(func_str(*args)) + else: + return self.add_element(structures["call"](func, args)) + + def visit_conditional_statement(self, n, vc): + return self.add_element(structures["if_then_else"]( + self.elements[vc[2]], + self.elements[vc[6]], + self.elements[vc[10]])) + + def visit_reference(self, n, vc): + id = self.add_element(structures["reference"]( + vc[0].lower().replace(" ", "_").strip("\""), self.subs)) + self.subs = None + return id + + def visit_array(self, n, vc): + if ";" in n.text or "," in n.text: + return self.add_element(np.squeeze(np.array( + [row.split(",") for row in n.text.strip(";").split(";")], + dtype=float))) + else: + return self.add_element(eval(n.text)) + + def visit_subscript_list(self, n, vc): + subs = [x.strip().replace("_", " ") for x in vc[2].split(",")] + self.subs = structures["subscripts_ref"](subs) + return "" + + def visit_name(self, n, vc): + return n.text.strip() + + def visit_expr(self, n, vc): + if vc[0] not in self.elements: + return self.add_element(eval(vc[0])) + else: + return vc[0] + + def visit_arguments(self, n, vc): + arglist = tuple(x.strip(",") for x in vc) + return self.add_element(tuple( + self.elements[arg] if arg in self.elements + else eval(arg) for arg in arglist)) + + def visit_parens(self, n, vc): + return vc[2] + + def visit__(self, n, vc): + # handles whitespace characters + return "" + + def generic_visit(self, n, vc): + return "".join(filter(None, vc)) or n.text + + def add_element(self, element): + return vu.add_element(self.elements, element) diff --git a/pysd/translators/xmile/xmile_file.py b/pysd/translators/xmile/xmile_file.py new file mode 100644 index 00000000..456cb091 --- /dev/null +++ b/pysd/translators/xmile/xmile_file.py @@ -0,0 +1,123 @@ +""" +The XmileFile class allows reading the original Xmile model file and +parsing it into Section elements. The final result can be exported to an +AbstractModel class in order to build a model in another programming language. +""" +from typing import Union +from pathlib import Path +from lxml import etree + +from ..structures.abstract_model import AbstractModel + +from .xmile_section import Section +from .xmile_utils import supported_extensions + + +class XmileFile(): + """ + The XmileFile class allows parsing an Xmile file. + When the object is created, the model file is automatically opened + and parsed with lxml.etree. + + Parameters + ---------- + xmile_path: str or pathlib.Path + Path to the Xmile model. + + """ + def __init__(self, xmile_path: Union[str, Path]): + self.xmile_path = Path(xmile_path) + self.root_path = self.xmile_path.parent + self.xmile_root = self._get_root() + self.ns = self.xmile_root.nsmap[None] # namespace of the xmile + self.view_elements = None + + def __str__(self): # pragma: no cover + return "\nXmile model file, loaded from:\n\t%s\n" % self.xmile_path + + @property + def _verbose(self) -> str: # pragma: no cover + """Get model information.""" + text = self.__str__() + for section in self.sections: + text += section._verbose + + return text + + @property + def verbose(self): # pragma: no cover + """Print model information to standard output.""" + print(self._verbose) + + def _get_root(self) -> etree._Element: + """ + Read an Xmile file and assign its content to self.model_text + + Returns + ------- + lxml.etree._Element: parsed xml object + + """ + # check for model extension + if self.xmile_path.suffix.lower() not in supported_extensions: + raise ValueError( + "The file to translate, '%s' " % self.xmile_path + + "is not a Xmile model. It must end with any of " + + "%s extensions." % ', '.join(supported_extensions) + ) + + return etree.parse( + str(self.xmile_path), + parser=etree.XMLParser(encoding="utf-8", recover=True) + ).getroot() + + def parse(self, parse_all: bool = True) -> None: + """ + Create a XmileSection object from the model content and parse it. + As macros are currently not supported, all models will + have a single section. This function should split the macros in + independent sections in the future. + + Parameters + ---------- + parse_all: bool (optional) + If True, the created XmileSection objects will be + automatically parsed. Otherwise, these objects will only be + added to self.sections but not parsed. Default is True. + + """ + # TODO: in order to make macros work we need to split them here + # in several sections + # We keep everything in a single section + self.sections = [Section( + name="__main__", + path=self.xmile_path.with_suffix(".py"), + section_type="main", + params=[], + returns=[], + content_root=self.xmile_root, + namespace=self.ns, + split=False, + views_dict=None)] + + if parse_all: + for section in self.sections: + section.parse() + + def get_abstract_model(self) -> AbstractModel: + """ + Get Abstract Model used for building. This, method should be + called after parsing the model (self.parse). This automatically + calls the get_abstract_section method from the model sections. + + Returns + ------- + AbstractModel: AbstractModel + Abstract Model object that can be used for building the model + in another language. + + """ + return AbstractModel( + original_path=self.xmile_path, + sections=tuple(section.get_abstract_section() + for section in self.sections)) diff --git a/pysd/translators/xmile/xmile_section.py b/pysd/translators/xmile/xmile_section.py new file mode 100644 index 00000000..45859bf6 --- /dev/null +++ b/pysd/translators/xmile/xmile_section.py @@ -0,0 +1,255 @@ +""" +The Section class allows parsing a model section into Elements. The +final result can be exported to an AbstractSection class in order to +build a model in other language. A section could be either the main model +(without the macros), or a macro definition (not supported yet for Xmile). +""" +from typing import List, Union +from lxml import etree +from pathlib import Path + +from ..structures.abstract_model import AbstractSection + +from .xmile_element import ControlElement, SubscriptRange, Flaux, Gf, Stock + + +class Section(): + """ + Section object allows parsing the elements of that section. + + Parameters + ---------- + name: str + Section name. '__main__' for the main section or the macro name. + + path: pathlib.Path + Section path. It should be the model name for main section and + the clean macro name for a macro. + + section_type: str ('main' or 'macro') + The section type. + + params: list + List of params that takes the section. In the case of main + section it will be an empty list. + + returns: list + List of variables that returns the section. In the case of main + section it will be an empty list. + + content_root: etree._Element + Section parsed tree content. + + namespace: str + The namespace of the section given after parsing its content + with etree. + + split: bool + If split is True the created section will split the variables + depending on the views_dict. + + views_dict: dict + The dictionary of the views. Giving the variables classified at + any level in order to split them by files. + + """ + _control_vars = ["initial_time", "final_time", "time_step", "saveper"] + + def __init__(self, name: str, path: Path, section_type: str, + params: List[str], returns: List[str], + content_root: etree._Element, namespace: str, split: bool, + views_dict: Union[dict, None]): + self.name = name + self.path = path + self.type = section_type + self.params = params + self.returns = returns + self.content = content_root + self.ns = {"ns": namespace} + self.split = split + self.views_dict = views_dict + self.elements = None + + def __str__(self): # pragma: no cover + return "\nSection: %s\n" % self.name + + @property + def _verbose(self) -> str: # pragma: no cover + """Get section information.""" + text = self.__str__() + if self.elements: + for element in self.elements: + text += element._verbose + else: + text += self.content + + return text + + @property + def verbose(self): # pragma: no cover + """Print section information to standard output.""" + print(self._verbose) + + def parse(self, parse_all: bool = True) -> None: + """ + Parse section object. The subscripts of the section will be added + to self subscripts. The variables defined as Flows, Auxiliary, Gf, + and Stock will be converted in XmileElements. The control variables, + if the section is __main__, will be converted to a ControlElement. + + Parameters + ---------- + parse_all: bool (optional) + If True then the created VensimElement objects will be + automatically parsed. Otherwise, this objects will only be + added to self.elements but not parser. Default is True. + + """ + # parse subscripts and components + self.subscripts = self._parse_subscripts() + self.components = self._parse_components() + + if self.name == "__main__": + # parse control variables + self.components += self._parse_control_vars() + + if parse_all: + [component.parse() for component in self.components] + + # define elements for printting information + self.elements = self.subscripts + self.components + + def _parse_subscripts(self) -> List[SubscriptRange]: + """Parse the subscripts of the section.""" + subscripts = [ + SubscriptRange( + node.attrib["name"], + [ + sub.attrib["name"] + for sub in node.xpath("ns:elem", namespaces=self.ns) + ], + []) # no subscript mapping implemented + for node + in self.content.xpath("ns:dimensions/ns:dim", namespaces=self.ns) + ] + self.subscripts_dict = { + subr.name: subr.definition for subr in subscripts} + return subscripts + + def _parse_components(self) -> List[Union[Flaux, Gf, Stock]]: + """ + Parse model components. Three groups defined: + Flaux: flows and auxiliary variables + Gf: lookups + Stock: integs + + """ + # Add flows and auxiliary variables + components = [ + Flaux(node, self.ns, self.subscripts_dict) + for node in self.content.xpath( + "ns:model/ns:variables/ns:aux|ns:model/ns:variables/ns:flow", + namespaces=self.ns) + if node.attrib["name"].lower().replace(" ", "_") + not in self._control_vars] + + # Add lookups + components += [ + Gf(node, self.ns, self.subscripts_dict) + for node in self.content.xpath( + "ns:model/ns:variables/ns:gf", + namespaces=self.ns) + ] + + # Add stocks + components += [ + Stock(node, self.ns, self.subscripts_dict) + for node in self.content.xpath( + "ns:model/ns:variables/ns:stock", + namespaces=self.ns) + ] + + return components + + def _parse_control_vars(self) -> List[ControlElement]: + """Parse control vars and rename them with Vensim standard.""" + + # Read the start time of simulation + node = self.content.xpath('ns:sim_specs', namespaces=self.ns)[0] + time_units = node.attrib['time_units']\ + if 'time_units' in node.attrib else "" + + control_vars = [] + + # initial time of the simulation + control_vars.append(ControlElement( + name="INITIAL TIME", + units=time_units, + documentation="The initial time for the simulation.", + eqn=node.xpath("ns:start", namespaces=self.ns)[0].text + )) + + # final time of the simulation + control_vars.append(ControlElement( + name="FINAL TIME", + units=time_units, + documentation="The final time for the simulation.", + eqn=node.xpath("ns:stop", namespaces=self.ns)[0].text + )) + + # time step of simulation + dt_node = node.xpath("ns:dt", namespaces=self.ns)[0] + dt_eqn = "1/(" + dt_node.text + ")" if "reciprocal" in dt_node.attrib\ + and dt_node.attrib["reciprocal"].lower() == "true"\ + else dt_node.text + + control_vars.append(ControlElement( + name="TIME STEP", + units=time_units, + documentation="The time step for the simulation.", + eqn=dt_eqn + )) + + # saving time of the simulation = time step + control_vars.append(ControlElement( + name="SAVEPER", + units=time_units, + documentation="The save time step for the simulation.", + eqn="time_step" + )) + + return control_vars + + def get_abstract_section(self) -> AbstractSection: + """ + Get Abstract Section used for building. This, method should be + called after parsing the section (self.parse). This method is + automatically called by Model's get_abstract_model and + automatically generates the AbstractSubscript ranges and merge + the components in elements calling also the get_abstract_components + method from each model component. + + Returns + ------- + AbstractSection: AbstractSection + Abstract Section object that can be used for building the model + in another language. + + """ + return AbstractSection( + name=self.name, + path=self.path, + type=self.type, + params=self.params, + returns=self.returns, + subscripts=[ + subs_range.get_abstract_subscript_range() + for subs_range in self.subscripts + ], + elements=[ + element.get_abstract_element() + for element in self.components + ], + split=self.split, + views_dict=self.views_dict + ) diff --git a/pysd/translators/xmile/xmile_structures.py b/pysd/translators/xmile/xmile_structures.py new file mode 100644 index 00000000..01fe23e0 --- /dev/null +++ b/pysd/translators/xmile/xmile_structures.py @@ -0,0 +1,88 @@ +import re +from ..structures import abstract_expressions as ae + + +structures = { + "reference": ae.ReferenceStructure, + "subscripts_ref": ae.SubscriptsReferenceStructure, + "arithmetic": ae.ArithmeticStructure, + "logic": ae.LogicStructure, + "inline_lookup": ae.InlineLookupsStructure, + "lookup": ae.LookupsStructure, + "call": ae.CallStructure, + "init": ae.InitialStructure, + "stock": ae.IntegStructure, + "delay1": { + 2: lambda x, y: ae.DelayStructure(x, y, x, 1), + 3: lambda x, y, z: ae.DelayStructure(x, y, z, 1) + }, + "delay3": { + 2: lambda x, y: ae.DelayStructure(x, y, x, 3), + 3: lambda x, y, z: ae.DelayStructure(x, y, z, 3), + }, + "delayn": { + 3: lambda x, y, n: ae.DelayNStructure(x, y, x, n), + 4: lambda x, y, n, z: ae.DelayNStructure(x, y, z, n), + }, + "smth1": { + 2: lambda x, y: ae.SmoothStructure(x, y, x, 1), + 3: lambda x, y, z: ae.SmoothStructure(x, y, z, 1) + }, + "smth3": { + 2: lambda x, y: ae.SmoothStructure(x, y, x, 3), + 3: lambda x, y, z: ae.SmoothStructure(x, y, z, 3) + }, + "smthn": { + 3: lambda x, y, n: ae.SmoothNStructure(x, y, x, n), + 4: lambda x, y, n, z: ae.SmoothNStructure(x, y, z, n) + }, + "trend": { + 2: lambda x, y: ae.TrendStructure(x, y, 0), + 3: ae.TrendStructure, + }, + "forcst": { + 3: lambda x, y, z: ae.ForecastStructure(x, y, z, 0), + 4: ae.ForecastStructure + }, + "safediv": { + 2: lambda x, y: ae.CallStructure( + ae.ReferenceStructure("zidz"), (x, y)), + 3: lambda x, y, z: ae.CallStructure( + ae.ReferenceStructure("xidz"), (x, y, z)) + }, + "if_then_else": lambda x, y, z: ae.CallStructure( + ae.ReferenceStructure("if_then_else"), (x, y, z)), + "ramp": { + 2: lambda x, y: ae.CallStructure( + ae.ReferenceStructure("Xramp"), (x, y)), + 3: lambda x, y, z: ae.CallStructure( + ae.ReferenceStructure("ramp"), (x, y, z)) + }, + "pulse": { + 2: lambda magnitude, start: ae.CallStructure( + ae.ReferenceStructure("Xpulse"), (start, magnitude)), + 3: lambda magnitude, start, interval: ae.CallStructure( + ae.ReferenceStructure("Xpulse_train"), (start, interval, magnitude) + ) + }, + "negative": lambda x: ae.ArithmeticStructure(["negative"], (x,)), + "int": lambda x: ae.CallStructure( + ae.ReferenceStructure("integer"), (x,)) +} + + +operators = { + "logic_ops": ["and", "or"], + "not_ops": ["not"], + "comp_ops": ["=", "<>", "<=", "<", ">=", ">"], + "add_ops": ["+", "-"], + "prod_ops": ["*", "/"], + "exp_ops": ["^"], + "pre_ops": ["+", "-"] +} + + +parsing_ops = { + key: "|".join(re.escape(x) for x in values) + for key, values in operators.items() +} diff --git a/pysd/translators/xmile/xmile_utils.py b/pysd/translators/xmile/xmile_utils.py new file mode 100644 index 00000000..eb7613a8 --- /dev/null +++ b/pysd/translators/xmile/xmile_utils.py @@ -0,0 +1,130 @@ +import re +import uuid + +import parsimonious +from typing import Dict +from pathlib import Path + + +supported_extensions = [".xmile", ".xml", ".stmx"] + + +class Grammar(): + _common_grammar = None + _grammar_path: Path = Path(__file__).parent.joinpath("parsing_grammars") + _grammar: Dict = {} + + @classmethod + def get(cls, grammar: str, subs: dict = {}) -> parsimonious.Grammar: + """Get parsimonious grammar for parsing""" + if grammar not in cls._grammar: + # include grammar in the class singleton + cls._grammar[grammar] = parsimonious.Grammar( + cls._read_grammar(grammar) % subs + ) + + return cls._grammar[grammar] + + @classmethod + def _read_grammar(cls, grammar: str) -> str: + """Read grammar from a file and include common grammar""" + with cls._gpath(grammar).open(encoding="ascii") as gfile: + source_grammar: str = gfile.read() + + return source_grammar + + @classmethod + def _gpath(cls, grammar: str) -> Path: + """Get the grammar file path""" + return cls._grammar_path.joinpath(grammar).with_suffix(".peg") + + +def split_arithmetic(structure: object, parsing_ops: dict, + expression: str, elements: dict, + negatives: set = set()) -> object: + """ + Split arithmetic pattern and return the corresponding object. + + Parameters + ---------- + structure: callable + Callable that generates the arithmetic object to return. + parsing_ops: dict + The parsing operators dictionary. + expression: str + Original expression with the operator and the hex code to the objects. + elements: dict + Dictionary of the hex identifiers and the objects that represent. + negative: set + Set of element hex values that must change their sign. + + Returns + ------- + object: structure + Final object of the arithmetic operation or initial object if + no operations are performed. + + """ + pattern = re.compile(parsing_ops) + parts = pattern.split(expression) # list of elements ids + ops = pattern.findall(expression) # operators list + ops = list(map( + lambda x: x.replace('and', ':AND:').replace('or', ':OR:'), ops)) + if not ops: + # no operators return original object + if parts[0] in negatives: + # make original object negative + negatives.remove(parts[0]) + return add_element( + elements, + structure(["negative"], (elements[parts[0]],))) + else: + return expression + else: + if not negatives: + # create arithmetic object + return add_element( + elements, + structure( + ops, + tuple([elements[id] for id in parts]))) + else: + # manage negative expressions + current_id = parts.pop() + current = elements[current_id] + if current_id in negatives: + negatives.remove(current_id) + current = structure(["negative"], (current,)) + while ops: + current_id = parts.pop() + current = structure( + [ops.pop()], + (elements[current_id], current)) + if current_id in negatives: + negatives.remove(current_id) + current = structure(["negative"], (current,)) + + return add_element(elements, current) + + +def add_element(elements: dict, element: object) -> str: + """ + Add element to elements dict using an unique hex identifier + + Parameters + ---------- + elements: dict + Dictionary of all elements. + + element: object + Element to add. + + Returns + ------- + id: str (hex) + The name of the key where element is saved in elements. + + """ + id = uuid.uuid4().hex + elements[id] = element + return id diff --git a/setup.py b/setup.py index 59750b4b..5bca76a9 100755 --- a/setup.py +++ b/setup.py @@ -29,8 +29,8 @@ ], install_requires=open('requirements.txt').read().strip().split('\n'), package_data={ - 'translation': [ - 'xmile/smile.grammar' + 'translators': [ + '*/parsing_grammars/*.peg' ] }, include_package_data=True diff --git a/tests/conftest.py b/tests/conftest.py index a6be8df1..b4fd902b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -8,6 +8,12 @@ def _root(): return Path(__file__).parent.resolve() +@pytest.fixture(scope="session") +def _test_models(_root): + # test-models directory + return _root.joinpath("test-models/tests") + + @pytest.fixture(scope="class") def shared_tmpdir(tmpdir_factory): # shared temporary directory for each class diff --git a/tests/integration_test_factory.py b/tests/integration_test_factory.py index ea85aeab..7ad08458 100644 --- a/tests/integration_test_factory.py +++ b/tests/integration_test_factory.py @@ -1,53 +1,32 @@ -from __future__ import print_function +import os.path +import glob -run = False - -if run: - - - import os.path - import textwrap - import glob - from pysd import utils - - test_dir = 'test-models/' - vensim_test_files = glob.glob(test_dir+'tests/*/*.mdl') +if False: + vensim_test_files = glob.glob("test-models/tests/*/*.mdl") + vensim_test_files.sort() tests = [] for file_path in vensim_test_files: - (path, file_name) = os.path.split(file_path) - (name, ext) = os.path.splitext(file_name) - - test_name = utils.make_python_identifier(path.split('/')[-1])[0] + path, file_name = os.path.split(file_path) + folder = path.split("/")[-1] test_func_string = """ - def test_%(test_name)s(self): - output, canon = runner('%(file_path)s') - assert_frames_close(output, canon, rtol=rtol) - """ % { - 'file_path': file_path, - 'test_name': test_name, + "%(test_name)s": { + "folder": "%(folder)s", + "file": "%(file_name)s" + },""" % { + "folder": folder, + "test_name": folder, + "file_name": file_name, } tests.append(test_func_string) - file_string = textwrap.dedent(''' - """ - Note that this file is autogenerated by `integration_test_factory.py` - and changes are likely to be overwritten. - """ - - import unittest - from pysd.tools.benchmarking import runner, assert_frames_close - - rtol = .05 - - - class TestIntegrationExamples(unittest.TestCase): - %(tests)s - - ''' % {'tests': ''.join(tests)}) + file_string = """ + vensim_test = {%(tests)s + } + """ % {"tests": "".join(tests)} - with open('integration_test_pysd.py', 'w', encoding='UTF-8') as ofile: + with open("test_factory_result.py", "w", encoding="UTF-8") as ofile: ofile.write(file_string) - print('generated %i integration tests' % len(tests)) + print("Generated %i integration tests" % len(tests)) diff --git a/tests/integration_test_vensim_pathway.py b/tests/integration_test_vensim_pathway.py deleted file mode 100644 index e2efaff2..00000000 --- a/tests/integration_test_vensim_pathway.py +++ /dev/null @@ -1,543 +0,0 @@ - -""" -Note that this file is autogenerated by `integration_test_factory.py` -and changes are likely to be overwritten. -""" -import os -import warnings -import unittest -from pysd.tools.benchmarking import runner, assert_frames_close - -rtol = .05 - -_root = os.path.dirname(__file__) -test_models = os.path.join(_root, "test-models/tests") - - -class TestIntegrationExamples(unittest.TestCase): - - def test_abs(self): - output, canon = runner(test_models + '/abs/test_abs.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_active_initial(self): - output, canon = runner(test_models + '/active_initial/test_active_initial.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_active_initial_circular(self): - output, canon = runner(test_models + '/active_initial_circular/test_active_initial_circular.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_arguments(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/arguments/test_arguments.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_array_with_line_break(self): - output, canon = runner(test_models + '/array_with_line_break/test_array_with_line_break.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_builtin_max(self): - output, canon = runner(test_models + '/builtin_max/builtin_max.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_builtin_min(self): - output, canon = runner(test_models + '/builtin_min/builtin_min.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_chained_initialization(self): - output, canon = runner(test_models + '/chained_initialization/test_chained_initialization.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip("Working on it") - def test_conditional_subscripts(self): - output, canon = runner(test_models + '/conditional_subscripts/test_conditional_subscripts.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_control_vars(self): - output, canon = runner(test_models + '/control_vars/test_control_vars.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_constant_expressions(self): - output, canon = runner(test_models + '/constant_expressions/test_constant_expressions.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_data_from_other_model(self): - output, canon = runner( - test_models + '/data_from_other_model/test_data_from_other_model.mdl', - data_files=test_models + '/data_from_other_model/data.tab') - assert_frames_close(output, canon, rtol=rtol) - - def test_delay_fixed(self): - # issue https://github.com/JamesPHoughton/pysd/issues/147 - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/delay_fixed/test_delay_fixed.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_delay_numeric_error(self): - # issue https://github.com/JamesPHoughton/pysd/issues/225 - output, canon = runner(test_models + '/delay_numeric_error/test_delay_numeric_error.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_delay_parentheses(self): - output, canon = runner(test_models + '/delay_parentheses/test_delay_parentheses.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_delay_pipeline(self): - # issue https://github.com/JamesPHoughton/pysd/issues/147 - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/delay_pipeline/test_pipeline_delays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_delays(self): - # issue https://github.com/JamesPHoughton/pysd/issues/147 - output, canon = runner(test_models + '/delays/test_delays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_dynamic_final_time(self): - # issue https://github.com/JamesPHoughton/pysd/issues/278 - output, canon = runner(test_models + '/dynamic_final_time/test_dynamic_final_time.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_euler_step_vs_saveper(self): - output, canon = runner(test_models + '/euler_step_vs_saveper/test_euler_step_vs_saveper.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_exp(self): - output, canon = runner(test_models + '/exp/test_exp.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_exponentiation(self): - output, canon = runner(test_models + '/exponentiation/exponentiation.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_forecast(self): - output, canon = runner(test_models + '/forecast/test_forecast.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_function_capitalization(self): - output, canon = runner(test_models + '/function_capitalization/test_function_capitalization.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_game(self): - output, canon = runner(test_models + '/game/test_game.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_get_constants_subrange(self): - output, canon = runner( - test_models + '/get_constants_subranges/' - + 'test_get_constants_subranges.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_data_args_3d_xls(self): - """ - Test for usage of GET DIRECT/XLS DATA with arguments from a Excel file - All the possible combinations of lentgh-wise and different dimensions - are tested in unit_test_external.py, this test want to test only the - good working of the builder - """ - output, canon = runner( - test_models + '/get_data_args_3d_xls/' - + 'test_get_data_args_3d_xls.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_lookups_data_3d_xls(self): - """ - Test for usage of GET DIRECT/XLS LOOKUPS/DATA from a Excel file - All the possible combinations of lentgh-wise and different dimensions - are tested in unit_test_external.py, this test want to test only the - good working of the builder - """ - output, canon = runner( - test_models + '/get_lookups_data_3d_xls/' - + 'test_get_lookups_data_3d_xls.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_lookups_subscripted_args(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner( - test_models + '/get_lookups_subscripted_args/' - + 'test_get_lookups_subscripted_args.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_lookups_subset(self): - output, canon = runner( - test_models + '/get_lookups_subset/' - + 'test_get_lookups_subset.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_with_missing_values_xlsx(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner( - test_models + '/get_with_missing_values_xlsx/' - + 'test_get_with_missing_values_xlsx.mdl' - ) - - assert_frames_close(output, canon, rtol=rtol) - - def test_get_mixed_definitions(self): - output, canon = runner( - test_models + '/get_mixed_definitions/' - + 'test_get_mixed_definitions.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_subscript_3d_arrays_xls(self): - """ - Test for usage of GET DIRECT/XLS SUBSCRIPTS/CONSTANTS from a Excel file - All the possible combinations of lentgh-wise and different dimensions - are tested in unit_test_external.py, this test want to test only the - good working of the builder - """ - output, canon = runner( - test_models + '/get_subscript_3d_arrays_xls/' - + 'test_get_subscript_3d_arrays_xls.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_get_xls_cellrange(self): - output, canon = runner( - test_models + '/get_xls_cellrange/' - + 'test_get_xls_cellrange.mdl' - ) - assert_frames_close(output, canon, rtol=rtol) - - def test_if_stmt(self): - output, canon = runner(test_models + '/if_stmt/if_stmt.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_initial_function(self): - output, canon = runner(test_models + '/initial_function/test_initial.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_input_functions(self): - output, canon = runner(test_models + '/input_functions/test_inputs.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_round(self): - output, canon = runner(test_models + '/subscripted_round/test_subscripted_round.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_invert_matrix(self): - output, canon = runner(test_models + '/invert_matrix/test_invert_matrix.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_limits(self): - output, canon = runner(test_models + '/limits/test_limits.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_line_breaks(self): - output, canon = runner(test_models + '/line_breaks/test_line_breaks.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_line_continuation(self): - output, canon = runner(test_models + '/line_continuation/test_line_continuation.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_ln(self): - output, canon = runner(test_models + '/ln/test_ln.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_log(self): - output, canon = runner(test_models + '/log/test_log.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_logicals(self): - output, canon = runner(test_models + '/logicals/test_logicals.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups(self): - output, canon = runner(test_models + '/lookups/test_lookups.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_without_range(self): - output, canon = runner(test_models + '/lookups_without_range/test_lookups_without_range.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_funcnames(self): - output, canon = runner(test_models + '/lookups_funcnames/test_lookups_funcnames.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_inline(self): - output, canon = runner(test_models + '/lookups_inline/test_lookups_inline.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_inline_bounded(self): - output, canon = runner(test_models + '/lookups_inline_bounded/test_lookups_inline_bounded.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_with_expr(self): - output, canon = runner(test_models + '/lookups_with_expr/test_lookups_with_expr.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_macro_cross_reference(self): - output, canon = runner(test_models + '/macro_cross_reference/test_macro_cross_reference.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_macro_expression(self): - output, canon = runner(test_models + '/macro_expression/test_macro_expression.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_macro_multi_expression(self): - output, canon = runner(test_models + '/macro_multi_expression/test_macro_multi_expression.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_macro_multi_macros(self): - output, canon = runner(test_models + '/macro_multi_macros/test_macro_multi_macros.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('working') - def test_macro_output(self): - output, canon = runner(test_models + '/macro_output/test_macro_output.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_macro_stock(self): - output, canon = runner(test_models + '/macro_stock/test_macro_stock.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('do we need this?') - def test_macro_trailing_definition(self): - output, canon = runner(test_models + '/macro_trailing_definition/test_macro_trailing_definition.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_model_doc(self): - output, canon = runner(test_models + '/model_doc/model_doc.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_nested_functions(self): - output, canon = runner(test_models + '/nested_functions/test_nested_functions.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_number_handling(self): - output, canon = runner(test_models + '/number_handling/test_number_handling.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_parentheses(self): - output, canon = runner(test_models + '/parentheses/test_parens.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('low priority') - def test_reference_capitalization(self): - """A properly formatted Vensim model should never create this failure""" - output, canon = runner(test_models + '/reference_capitalization/test_reference_capitalization.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_repeated_subscript(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/repeated_subscript/test_repeated_subscript.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_rounding(self): - output, canon = runner(test_models + '/rounding/test_rounding.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_sample_if_true(self): - output, canon = runner(test_models + '/sample_if_true/test_sample_if_true.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_smooth(self): - output, canon = runner(test_models + '/smooth/test_smooth.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_smooth_and_stock(self): - output, canon = runner(test_models + '/smooth_and_stock/test_smooth_and_stock.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_special_characters(self): - output, canon = runner(test_models + '/special_characters/test_special_variable_names.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_sqrt(self): - output, canon = runner(test_models + '/sqrt/test_sqrt.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subrange_merge(self): - output, canon = runner(test_models + '/subrange_merge/test_subrange_merge.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_logicals(self): - output, canon = runner(test_models + '/subscript_logicals/test_subscript_logicals.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_multiples(self): - output, canon = runner(test_models + '/subscript_multiples/test_multiple_subscripts.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_1d_arrays(self): - output, canon = runner(test_models + '/subscript_1d_arrays/test_subscript_1d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_2d_arrays(self): - output, canon = runner(test_models + '/subscript_2d_arrays/test_subscript_2d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_3d_arrays(self): - output, canon = runner(test_models + '/subscript_3d_arrays/test_subscript_3d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_3d_arrays_lengthwise(self): - output, canon = runner(test_models + '/subscript_3d_arrays_lengthwise/test_subscript_3d_arrays_lengthwise.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_3d_arrays_widthwise(self): - output, canon = runner(test_models + '/subscript_3d_arrays_widthwise/test_subscript_3d_arrays_widthwise.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_aggregation(self): - output, canon = runner(test_models + '/subscript_aggregation/test_subscript_aggregation.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_constant_call(self): - output, canon = runner(test_models + '/subscript_constant_call/test_subscript_constant_call.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_copy(self): - output, canon = runner(test_models + '/subscript_copy/test_subscript_copy.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_docs(self): - output, canon = runner(test_models + '/subscript_docs/subscript_docs.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_element_name(self): - # issue https://github.com/JamesPHoughton/pysd/issues/216 - output, canon = runner(test_models + '/subscript_element_name/test_subscript_element_name.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_individually_defined_1_of_2d_arrays(self): - output, canon = runner(test_models + '/subscript_individually_defined_1_of_2d_arrays/subscript_individually_defined_1_of_2d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_individually_defined_1_of_2d_arrays_from_floats(self): - output, canon = runner(test_models + '/subscript_individually_defined_1_of_2d_arrays_from_floats/subscript_individually_defined_1_of_2d_arrays_from_floats.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_individually_defined_1d_arrays(self): - output, canon = runner(test_models + '/subscript_individually_defined_1d_arrays/subscript_individually_defined_1d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_individually_defined_stocks(self): - output, canon = runner(test_models + '/subscript_individually_defined_stocks/test_subscript_individually_defined_stocks.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_mapping_simple(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/subscript_mapping_simple/test_subscript_mapping_simple.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_mapping_vensim(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore") - output, canon = runner(test_models + '/subscript_mapping_vensim/test_subscript_mapping_vensim.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_mixed_assembly(self): - output, canon = runner(test_models + '/subscript_mixed_assembly/test_subscript_mixed_assembly.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_selection(self): - output, canon = runner(test_models + '/subscript_selection/subscript_selection.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_numeric_range(self): - output, canon = runner(test_models + '/subscript_numeric_range/test_subscript_numeric_range.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_subranges(self): - output, canon = runner(test_models + '/subscript_subranges/test_subscript_subrange.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_subranges_equal(self): - output, canon = runner(test_models + '/subscript_subranges_equal/test_subscript_subrange_equal.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_switching(self): - output, canon = runner(test_models + '/subscript_switching/subscript_switching.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_transposition(self): - output, canon = runner(test_models + '/subscript_transposition/test_subscript_transposition.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscript_updimensioning(self): - output, canon = runner(test_models + '/subscript_updimensioning/test_subscript_updimensioning.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_delays(self): - output, canon = runner(test_models + '/subscripted_delays/test_subscripted_delays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_flows(self): - output, canon = runner(test_models + '/subscripted_flows/test_subscripted_flows.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_if_then_else(self): - output, canon = runner(test_models + '/subscripted_if_then_else/test_subscripted_if_then_else.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_logicals(self): - output, canon = runner(test_models + '/subscripted_logicals/test_subscripted_logicals.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_smooth(self): - # issue https://github.com/JamesPHoughton/pysd/issues/226 - output, canon = runner(test_models + '/subscripted_smooth/test_subscripted_smooth.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_trend(self): - # issue https://github.com/JamesPHoughton/pysd/issues/226 - output, canon = runner(test_models + '/subscripted_trend/test_subscripted_trend.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subscripted_xidz(self): - output, canon = runner(test_models + '/subscripted_xidz/test_subscripted_xidz.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_subset_duplicated_coord(self): - output, canon = runner(test_models + '/subset_duplicated_coord/' - + 'test_subset_duplicated_coord.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_time(self): - output, canon = runner(test_models + '/time/test_time.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_trend(self): - output, canon = runner(test_models + '/trend/test_trend.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_trig(self): - output, canon = runner(test_models + '/trig/test_trig.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_variable_ranges(self): - output, canon = runner(test_models + '/variable_ranges/test_variable_ranges.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_unicode_characters(self): - output, canon = runner(test_models + '/unicode_characters/unicode_test_model.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_xidz_zidz(self): - output, canon = runner(test_models + '/xidz_zidz/xidz_zidz.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_run_uppercase(self): - output, canon = runner(test_models + '/case_sensitive_extension/teacup-upper.MDL') - assert_frames_close(output, canon, rtol=rtol) - - def test_odd_number_quotes(self): - output, canon = runner(test_models + '/odd_number_quotes/teacup_3quotes.mdl') - assert_frames_close(output, canon, rtol=rtol) diff --git a/tests/integration_test_xmile_pathway.py b/tests/integration_test_xmile_pathway.py deleted file mode 100644 index 3fcfc1ad..00000000 --- a/tests/integration_test_xmile_pathway.py +++ /dev/null @@ -1,370 +0,0 @@ - -import os -import unittest -from pysd.tools.benchmarking import runner, assert_frames_close - -rtol = .05 - -_root = os.path.dirname(__file__) -test_models = os.path.join(_root, "test-models/tests") - - -class TestIntegrationExamples(unittest.TestCase): - - def test_abs(self): - output, canon = runner(test_models + '/abs/test_abs.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('error in model file') - def test_active_initial(self): - output, canon = runner(test_models + '/active_initial/test_active_initial.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing model file') - def test_arguments(self): - output, canon = runner(test_models + '/arguments/test_arguments.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_builtin_max(self): - output, canon = runner(test_models + '/builtin_max/builtin_max.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_builtin_min(self): - output, canon = runner(test_models + '/builtin_min/builtin_min.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_chained_initialization(self): - output, canon = runner( - test_models + '/chained_initialization/test_chained_initialization.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_comparisons(self): - output, canon = runner( - test_models + '/comparisons/comparisons.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_constant_expressions(self): - output, canon = runner( - test_models + '/constant_expressions/test_constant_expressions.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_delay_parentheses(self): - output, canon = runner( - test_models + '/delay_parentheses/test_delay_parentheses.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_delays(self): - output, canon = runner(test_models + '/delays/test_delays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_euler_step_vs_saveper(self): - output, canon = runner( - test_models + '/euler_step_vs_saveper/test_euler_step_vs_saveper.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_eval_order(self): - output, canon = runner( - test_models + '/eval_order/eval_order.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_exp(self): - output, canon = runner(test_models + '/exp/test_exp.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_exponentiation(self): - output, canon = runner(test_models + '/exponentiation/exponentiation.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_function_capitalization(self): - output, canon = runner( - test_models + '/function_capitalization/test_function_capitalization.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('not sure if this is implemented in xmile?') - def test_game(self): - output, canon = runner(test_models + '/game/test_game.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_if_stmt(self): - output, canon = runner(test_models + '/if_stmt/if_stmt.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_initial_function(self): - output, canon = runner(test_models + '/initial_function/test_initial.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile model') - def test_input_functions(self): - output, canon = runner(test_models + '/input_functions/test_inputs.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_limits(self): - output, canon = runner(test_models + '/limits/test_limits.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_line_breaks(self): - output, canon = runner(test_models + '/line_breaks/test_line_breaks.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_line_continuation(self): - output, canon = runner(test_models + '/line_continuation/test_line_continuation.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_ln(self): - output, canon = runner(test_models + '/ln/test_ln.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_log(self): - output, canon = runner(test_models + '/log/test_log.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_logicals(self): - output, canon = runner(test_models + '/logicals/test_logicals.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups(self): - output, canon = runner(test_models + '/lookups/test_lookups.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_xscale(self): - output, canon = runner(test_models + '/lookups/test_lookups_xscale.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_xpts_sep(self): - output, canon = runner(test_models + '/lookups/test_lookups_xpts_sep.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_ypts_sep(self): - output, canon = runner(test_models + '/lookups/test_lookups_ypts_sep.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_lookups_funcnames(self): - output, canon = runner(test_models + '/lookups_funcnames/test_lookups_funcnames.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_lookups_inline(self): - output, canon = runner(test_models + '/lookups_inline/test_lookups_inline.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_lookups_inline_bounded(self): - output, canon = runner( - test_models + '/lookups_inline_bounded/test_lookups_inline_bounded.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_macro_cross_reference(self): - output, canon = runner(test_models + '/macro_cross_reference/test_macro_cross_reference.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_macro_expression(self): - output, canon = runner(test_models + '/macro_expression/test_macro_expression.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_macro_multi_expression(self): - output, canon = runner( - test_models + '/macro_multi_expression/test_macro_multi_expression.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_macro_multi_macros(self): - output, canon = runner( - test_models + '/macro_multi_macros/test_macro_multi_macros.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_macro_output(self): - output, canon = runner(test_models + '/macro_output/test_macro_output.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_macro_stock(self): - output, canon = runner(test_models + '/macro_stock/test_macro_stock.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('do we need this?') - def test_macro_trailing_definition(self): - output, canon = runner(test_models + '/macro_trailing_definition/test_macro_trailing_definition.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_model_doc(self): - output, canon = runner(test_models + '/model_doc/model_doc.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_number_handling(self): - output, canon = runner(test_models + '/number_handling/test_number_handling.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_parentheses(self): - output, canon = runner(test_models + '/parentheses/test_parens.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_reference_capitalization(self): - """A properly formatted Vensim model should never create this failure""" - output, canon = runner( - test_models + '/reference_capitalization/test_reference_capitalization.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('in branch') - def test_rounding(self): - output, canon = runner(test_models + '/rounding/test_rounding.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_smooth(self): - output, canon = runner(test_models + '/smooth/test_smooth.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_smooth_and_stock(self): - output, canon = runner(test_models + '/smooth_and_stock/test_smooth_and_stock.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_special_characters(self): - output, canon = runner( - test_models + '/special_characters/test_special_variable_names.xmile') - assert_frames_close(output, canon, rtol=rtol) - - def test_sqrt(self): - output, canon = runner(test_models + '/sqrt/test_sqrt.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_multiples(self): - output, canon = runner( - test_models + '/subscript multiples/test_multiple_subscripts.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_1d_arrays(self): - output, canon = runner( - test_models + '/subscript_1d_arrays/test_subscript_1d_arrays.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_2d_arrays(self): - output, canon = runner( - test_models + '/subscript_2d_arrays/test_subscript_2d_arrays.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_3d_arrays(self): - output, canon = runner(test_models + '/subscript_3d_arrays/test_subscript_3d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_3d_arrays_lengthwise(self): - output, canon = runner(test_models + '/subscript_3d_arrays_lengthwise/test_subscript_3d_arrays_lengthwise.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_3d_arrays_widthwise(self): - output, canon = runner(test_models + '/subscript_3d_arrays_widthwise/test_subscript_3d_arrays_widthwise.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('in branch') - def test_subscript_aggregation(self): - output, canon = runner(test_models + '/subscript_aggregation/test_subscript_aggregation.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_constant_call(self): - output, canon = runner( - test_models + '/subscript_constant_call/test_subscript_constant_call.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_docs(self): - output, canon = runner(test_models + '/subscript_docs/subscript_docs.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_individually_defined_1_of_2d_arrays(self): - output, canon = runner(test_models + '/subscript_individually_defined_1_of_2d_arrays/subscript_individually_defined_1_of_2d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_individually_defined_1_of_2d_arrays_from_floats(self): - output, canon = runner(test_models + '/subscript_individually_defined_1_of_2d_arrays_from_floats/subscript_individually_defined_1_of_2d_arrays_from_floats.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_individually_defined_1d_arrays(self): - output, canon = runner(test_models + '/subscript_individually_defined_1d_arrays/subscript_individually_defined_1d_arrays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_individually_defined_stocks(self): - output, canon = runner(test_models + '/subscript_individually_defined_stocks/test_subscript_individually_defined_stocks.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_mixed_assembly(self): - output, canon = runner(test_models + '/subscript_mixed_assembly/test_subscript_mixed_assembly.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_selection(self): - output, canon = runner(test_models + '/subscript_selection/subscript_selection.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_subranges(self): - output, canon = runner( - test_models + '/subscript_subranges/test_subscript_subrange.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_subranges_equal(self): - output, canon = runner( - test_models + '/subscript_subranges_equal/test_subscript_subrange_equal.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscript_switching(self): - output, canon = runner(test_models + '/subscript_switching/subscript_switching.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('missing test model') - def test_subscript_updimensioning(self): - output, canon = runner( - test_models + '/subscript_updimensioning/test_subscript_updimensioning.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscripted_delays(self): - output, canon = runner(test_models + '/subscripted_delays/test_subscripted_delays.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_subscripted_flows(self): - output, canon = runner(test_models + '/subscripted_flows/test_subscripted_flows.mdl') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_time(self): - output, canon = runner(test_models + '/time/test_time.mdl') - assert_frames_close(output, canon, rtol=rtol) - - def test_trig(self): - output, canon = runner(test_models + '/trig/test_trig.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_trend(self): - output, canon = runner(test_models + '/trend/test_trend.xmile') - assert_frames_close(output, canon, rtol=rtol) - - @unittest.skip('no xmile') - def test_xidz_zidz(self): - output, canon = runner(test_models + '/xidz_zidz/xidz_zidz.xmile') - assert_frames_close(output, canon, rtol=rtol) - - diff --git a/tests/more-tests/circular_reference/test_circular_reference.py b/tests/more-tests/circular_reference/test_circular_reference.py index 9fd64ac3..17d822e9 100644 --- a/tests/more-tests/circular_reference/test_circular_reference.py +++ b/tests/more-tests/circular_reference/test_circular_reference.py @@ -1,63 +1,66 @@ from pysd.py_backend.statefuls import Integ, Delay +from pysd import Component _subscript_dict = {} -_namespace = {'integ': 'integ', 'delay': 'delay'} -_dependencies = { - 'integ': {'_integ_integ': 1}, - 'delay': {'_delay_delay': 1}, - '_integ_integ': {'initial': {'delay': 1}, 'step': {}}, - '_delay_delay': {'initial': {'integ': 1}, 'step': {}} -} -__pysd_version__ = "2.0.0" + +__pysd_version__ = "3.0.0" __data = {'scope': None, 'time': lambda: 0} +component = Component() + +_control_vars = { + "initial_time": lambda: 0, + "final_time": lambda: 0.5, + "time_step": lambda: 0.5, + "saveper": lambda: time_step(), +} + def _init_outer_references(data): for key in data: __data[key] = data[key] +@component.add(name="Time") def time(): return __data["time"]() -def _time_step(): - return 0.5 - - -def _initial_time(): - return 0 - - -def _final_time(): - return 0.5 - - -def _saveper(): - return 0.5 - - +@component.add(name="Time step") def time_step(): return __data["time"].step() +@component.add(name="Initial time") def initial_time(): return __data["time"].initial() +@component.add(name="Final time") def final_time(): return __data["time"].final() +@component.add(name="Saveper") def saveper(): return __data["time"].save() +@component.add( + name="Integ", + depends_on={'_integ_integ': 1}, + other_deps={'_integ_integ': {'initial': {'delay': 1}, 'step': {}}} +) def integ(): return _integ_integ() +@component.add( + name="Delay", + depends_on={'_delay_delay': 1}, + other_deps={'_delay_delay': {'initial': {'integ': 1}, 'step': {}}} +) def delay(): return _delay_delay() @@ -65,4 +68,4 @@ def delay(): _integ_integ = Integ(lambda: 2, lambda: delay(), '_integ_integ') _delay_delay = Delay(lambda: 2, lambda: 1, - lambda: integ(), 1, time_step, '_delay_delay') \ No newline at end of file + lambda: integ(), 1, time_step, '_delay_delay') diff --git a/tests/more-tests/initialization_order/test_initialization_order.py b/tests/more-tests/initialization_order/test_initialization_order.py index 8106907a..d132755b 100644 --- a/tests/more-tests/initialization_order/test_initialization_order.py +++ b/tests/more-tests/initialization_order/test_initialization_order.py @@ -5,37 +5,16 @@ from pysd.py_backend.statefuls import Integ +from pysd import Component -__pysd_version__ = "2.0.0" +__pysd_version__ = "3.0.0" _subscript_dict = {} -_namespace = { - "TIME": "time", - "Time": "time", - "Stock B": "stock_b", - "Stock A": "stock_a", - "Initial Parameter": "initial_parameter", - "FINAL TIME": "final_time", - "INITIAL TIME": "initial_time", - "SAVEPER": "saveper", - "TIME STEP": "time_step", -} - -_dependencies = { - 'initial_time': {}, - 'final_time': {}, - 'time_step': {}, - 'saveper': {'time_step': 1}, - 'initial_parameter': {}, - 'stock_a': {'_integ_stock_a': 1}, - 'stock_b': {'_integ_stock_b': 1}, - '_integ_stock_a': {'initial': {'initial_parameter': 1}, 'step': {}}, - '_integ_stock_b': {'initial': {'stock_a': 1}, 'step': {}} -} - __data = {"scope": None, "time": lambda: 0} +component = Component() + _control_vars = { "initial_time": lambda: 0, "final_time": lambda: 20, @@ -49,69 +28,57 @@ def _init_outer_references(data): __data[key] = data[key] +@component.add(name="Time") def time(): return __data["time"]() +@component.add(name="Initial time") def initial_time(): return __data["time"].initial_time() +@component.add(name="Final time") def final_time(): return __data["time"].final_time() +@component.add(name="Time step") def time_step(): return __data["time"].time_step() +@component.add(name="Saveper", depends_on={'time_step': 1}) def saveper(): return __data["time"].saveper() +@component.add(name="Stock B", depends_on={'_integ_stock_b': 1}, + other_deps={ + '_integ_stock_b': { + 'initial': {'stock_a': 1}, + 'step': {} + }}) def stock_b(): - """ - Real Name: Stock B - Original Eqn: INTEG(1, Stock A) - Units: - Limits: (None, None) - Type: component - Subs: None - - - """ return _integ_stock_b() +@component.add(name="Stock A", depends_on={'_integ_stock_a': 1}, + other_deps={ + '_integ_stock_a': { + 'initial': {'initial_par': 1}, + 'step': {} + }}) def stock_a(): - """ - Real Name: Stock A - Original Eqn: INTEG (1, Initial Parameter) - Units: - Limits: (None, None) - Type: component - Subs: None - - - """ return _integ_stock_a() -def initial_parameter(): - """ - Real Name: Initial Parameter - Original Eqn: 42 - Units: - Limits: (None, None) - Type: constant - Subs: None - - - """ +@component.add(name="Initial par") +def initial_par(): return 42 _integ_stock_b = Integ(lambda: 1, lambda: stock_a(), "_integ_stock_b") -_integ_stock_a = Integ(lambda: 1, lambda: initial_parameter(), "_integ_stock_a") +_integ_stock_a = Integ(lambda: 1, lambda: initial_par(), "_integ_stock_a") diff --git a/tests/more-tests/not_implemented_and_incomplete/test_not_implemented_and_incomplete.mdl b/tests/more-tests/not_implemented_and_incomplete/test_not_implemented_and_incomplete.mdl new file mode 100644 index 00000000..e814f29e --- /dev/null +++ b/tests/more-tests/not_implemented_and_incomplete/test_not_implemented_and_incomplete.mdl @@ -0,0 +1,36 @@ +{UTF-8} +incomplete var = A FUNCTION OF( Time) + ~ + ~ | + +not implemented function= + MY FUNC(Time) + ~ + ~ | + +******************************************************** + .Control +********************************************************~ + Simulation Control Parameters + | + +FINAL TIME = 1 + ~ Month + ~ The final time for the simulation. + | + +INITIAL TIME = 0 + ~ Month + ~ The initial time for the simulation. + | + +SAVEPER = + TIME STEP + ~ Month [0,?] + ~ The frequency with which output is stored. + | + +TIME STEP = 1 + ~ Month [0,?] + ~ The time step for the simulation. + | diff --git a/tests/more-tests/version/test_old_version.py b/tests/more-tests/old_version/test_old_version.py similarity index 97% rename from tests/more-tests/version/test_old_version.py rename to tests/more-tests/old_version/test_old_version.py index e833a17f..64b1f21d 100644 --- a/tests/more-tests/version/test_old_version.py +++ b/tests/more-tests/old_version/test_old_version.py @@ -1,6 +1,5 @@ __pysd_version__ = "1.5.0" -_namespace = {} _dependencies = {} __data = {'scope': None, 'time': lambda: 0} diff --git a/tests/more-tests/random/test_random.mdl b/tests/more-tests/random/test_random.mdl new file mode 100644 index 00000000..da30cb61 --- /dev/null +++ b/tests/more-tests/random/test_random.mdl @@ -0,0 +1,193 @@ +{UTF-8} +A B uniform matrix[Dims,Subs]= + RANDOM UNIFORM(10, 11, 1) + ~ + ~ | + +A B uniform matrix 1[Dims,Subs]= + RANDOM UNIFORM(my var[Dims]+100, 200, 1) + ~ + ~ | + +A B uniform matrix 1 0[Dims,Subs]= + RANDOM UNIFORM(my var2[Subs], my var[Dims], 1) + ~ + ~ | + +A B uniform scalar= + RANDOM UNIFORM(-1, 10, 1) + ~ + ~ | + +A B uniform vec[Dims]= + RANDOM UNIFORM(2, 3, 1) + ~ + ~ | + +A B uniform vec 1[Dims]= + RANDOM UNIFORM(my var[Dims], 100, 1) + ~ + ~ | + +Dims: + (Dim1-Dim25) + ~ + ~ | + +my var[Dims]= + Dims-50 + ~ + ~ | + +my var2[Subs]= + 2*Subs + ~ + ~ | + +normal A B uniform matrix[Dims,Subs]= + RANDOM NORMAL(-1, 10, 5, 1, 1) + ~ + ~ | + +normal A B uniform matrix 1[Dims,Subs]= + RANDOM NORMAL(my var[Dims], my var[Dims]+5, my var[Dims]+2, 10, 1) + ~ + ~ | + +normal A B uniform matrix 1 0[Dims,Subs]= + RANDOM NORMAL(my var[Dims], my var[Dims]+5, my var[Dims]+2, my var2[Subs], 1) + ~ + ~ | + +normal scalar= + RANDOM NORMAL(2, 4, 0.5, 2, 1) + ~ + ~ | + +normal vec[Dims]= + RANDOM NORMAL(1, 5, 3.5, 10, 1) + ~ + ~ | + +normal vec 1[Dims]= + RANDOM NORMAL(my var[Dims], 400, my var[Dims]+200, 2, 1) + ~ + ~ | + +Subs: + (sub1-sub50) + ~ + ~ | + +uniform matrix[Dims,Subs]= + RANDOM 0 1() + ~ + ~ | + +uniform scalar= + RANDOM 0 1() + ~ + ~ | + +uniform vec[Dims]= + RANDOM 0 1() + ~ + ~ | + +******************************************************** + .Control +********************************************************~ + Simulation Control Parameters + | + +FINAL TIME = 10 + ~ Month + ~ The final time for the simulation. + | + +INITIAL TIME = 0 + ~ Month + ~ The initial time for the simulation. + | + +SAVEPER = + TIME STEP + ~ Month [0,?] + ~ The frequency with which output is stored. + | + +TIME STEP = 1 + ~ Month [0,?] + ~ The time step for the simulation. + | + +\\\---/// Sketch information - do not modify anything except names +V300 Do not put anything below this section - it will be ignored +*View 1 +$192-192-192,0,Times New Roman|12||0-0-0|0-0-0|0-0-255|-1--1--1|-1--1--1|96,96,100,0 +10,1,uniform scalar,275,125,44,11,8,3,0,0,0,0,0,0 +10,2,uniform vec,254,206,37,11,8,3,0,0,0,0,0,0 +10,3,uniform matrix,352,338,45,11,8,3,0,0,0,0,0,0 +10,4,A B uniform scalar,524,129,59,11,8,3,0,0,0,0,0,0 +10,5,A B uniform vec,503,210,52,11,8,3,0,0,0,0,0,0 +10,6,A B uniform matrix,601,342,60,11,8,3,0,0,0,0,0,0 +10,7,A B uniform vec 1,733,205,58,11,8,3,0,0,0,0,0,0 +10,8,A B uniform matrix 1,831,337,40,19,8,3,0,0,0,0,0,0 +10,9,A B uniform matrix 1 0,967,340,40,19,8,3,0,0,0,0,0,0 +10,10,my var,949,87,23,11,8,3,0,0,0,0,0,0 +1,11,10,8,0,0,0,0,0,128,0,-1--1--1,,1|(894,201)| +1,12,10,7,0,0,0,0,0,128,0,-1--1--1,,1|(847,142)| +1,13,10,9,0,0,0,0,0,128,0,-1--1--1,,1|(956,202)| +10,14,my var2,1243,87,27,11,8,3,0,0,0,0,0,0 +1,15,14,9,0,0,0,0,0,128,0,-1--1--1,,1|(1114,204)| +10,16,normal scalar,1157,311,45,19,8,3,0,0,0,0,0,0 +10,17,normal vec,1162,373,37,19,8,3,0,0,0,0,0,0 +10,18,normal A B uniform matrix,1281,367,46,19,8,3,0,0,0,0,0,0 +10,19,normal vec 1,1188,262,44,19,8,3,0,0,0,0,0,0 +10,20,normal A B uniform matrix 1,1511,362,63,19,8,3,0,0,0,0,0,0 +10,21,normal A B uniform matrix 1 0,1647,365,63,19,8,3,0,0,0,0,0,0 +1,22,10,19,0,0,0,0,0,128,0,-1--1--1,,1|(1057,166)| +1,23,10,20,0,0,0,0,0,128,0,-1--1--1,,1|(1215,217)| +1,24,10,21,0,0,0,0,0,128,0,-1--1--1,,1|(1278,218)| +1,25,14,20,0,0,0,0,0,128,0,-1--1--1,,1|(1367,215)| +1,26,14,21,0,0,0,0,0,128,0,-1--1--1,,1|(1432,218)| +///---\\\ +:L<%^E!@ +1:Current.vdf +9:Current +15:0,0,0,0,0,0 +19:100,0 +27:0, +34:0, +4:Time +5:A B uniform matrix[Dims,Subs] +35:Date +36:YYYY-MM-DD +37:2000 +38:1 +39:1 +40:2 +41:0 +42:1 +24:0 +25:100 +26:100 +57:1 +54:0 +55:0 +59:0 +56:0 +58:0 +44:65001 +46:0 +45:1 +49:1 +50:0 +51: +52: +53: +43:output +47:Current +48: +6:Dim1 +6:sub1 diff --git a/tests/more-tests/split_model/input.xlsx b/tests/more-tests/split_model/input.xlsx new file mode 100644 index 00000000..2a828d5d Binary files /dev/null and b/tests/more-tests/split_model/input.xlsx differ diff --git a/tests/more-tests/split_model/test_split_model.mdl b/tests/more-tests/split_model/test_split_model.mdl index 99005d30..38551b71 100644 --- a/tests/more-tests/split_model/test_split_model.mdl +++ b/tests/more-tests/split_model/test_split_model.mdl @@ -1,49 +1,49 @@ {UTF-8} another var= 3*Stock - ~ + ~ ~ | initial stock= - 0.1 - ~ + GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'A1') + ~ ~ | initial stock correction= - 0 - ~ + GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'A2') + ~ ~ | lookup table( (1,0), (10,2), (100,2), (1000,3), (10000,4)) - ~ + ~ ~ | other stock= INTEG ( 6*"var-n", 3) - ~ + ~ ~ | "rate-1"= "var-n" - ~ + ~ ~ | Stock= INTEG ( "rate-1"+Time*initial stock correction, initial stock+initial stock correction) - ~ + ~ ~ | "var-n"= 5 - ~ + ~ ~ | "variable-x"= lookup table(6*another var) - ~ + ~ ~ | ******************************************************** @@ -62,7 +62,7 @@ INITIAL TIME = 0 ~ The initial time for the simulation. | -SAVEPER = +SAVEPER = TIME STEP ~ Month [0,?] ~ The frequency with which output is stored. diff --git a/tests/more-tests/split_model/test_split_model_sub_subviews.mdl b/tests/more-tests/split_model/test_split_model_sub_subviews.mdl index 74cf9fb2..9a90a4c1 100644 --- a/tests/more-tests/split_model/test_split_model_sub_subviews.mdl +++ b/tests/more-tests/split_model/test_split_model_sub_subviews.mdl @@ -2,63 +2,63 @@ other stock= INTEG ( var tolo, 0) - ~ + ~ ~ | interesting var 2 looked up= look up definition(interesting var 2) - ~ + ~ ~ | look up definition( (1,0), (10,1), (50,1.5), (100,4), (1000,5), (10000,3), (100000,4)) - ~ + ~ ~ | var tolo= 55+great var - ~ + ~ ~ | great var= 5 - ~ + ~ ~ | interesting var 1= "variable-x"+1 - ~ + ~ ~ | interesting var 2= interesting var 1*5 - ~ + ~ ~ | another var= 3*Stock - ~ + ~ ~ | "rate-1"= "var-n" - ~ + ~ ~ | "var-n"= 5 - ~ + ~ ~ | "variable-x"= - 6*another var - ~ + ACTIVE INITIAL(6*another var, 1) + ~ ~ | Stock= INTEG ( "rate-1", 1) - ~ + ~ ~ | ******************************************************** @@ -77,7 +77,7 @@ INITIAL TIME = 0 ~ The initial time for the simulation. | -SAVEPER = +SAVEPER = TIME STEP ~ Month [0,?] ~ The frequency with which output is stored. diff --git a/tests/more-tests/subscript_individually_defined_stocks2/test_subscript_individually_defined_stocks2.mdl b/tests/more-tests/subscript_individually_defined_stocks2/test_subscript_individually_defined_stocks2.mdl index aa30749e..e9470cf1 100644 --- a/tests/more-tests/subscript_individually_defined_stocks2/test_subscript_individually_defined_stocks2.mdl +++ b/tests/more-tests/subscript_individually_defined_stocks2/test_subscript_individually_defined_stocks2.mdl @@ -23,9 +23,9 @@ Third Dimension Subscript: ~ | Initial Values[One Dimensional Subscript,Second Dimension Subscript,Depth 1]= - Initial Values A ~~| + Initial Values A[One Dimensional Subscript,Second Dimension Subscript] ~~| Initial Values[One Dimensional Subscript,Second Dimension Subscript,Depth 2]= - Initial Values B + Initial Values B[One Dimensional Subscript,Second Dimension Subscript] ~ ~ | diff --git a/tests/more-tests/type_error/test_type_error.py b/tests/more-tests/type_error/test_type_error.py index f7da1b62..c44f18cf 100644 --- a/tests/more-tests/type_error/test_type_error.py +++ b/tests/more-tests/type_error/test_type_error.py @@ -1,6 +1,8 @@ -from pysd import external +from pysd import external, Component +__pysd_version__ = "3.0.0" _root = './' -external.ExtData('input.xlsx', 'Sheet1', '5', 'B6', - None, {}, [], _root, '_ext_data') \ No newline at end of file +component = Component() + +external.ExtData('input.xlsx', 'Sheet1', '5', 'B6') diff --git a/tests/more-tests/version/test_current_version.py b/tests/more-tests/version/test_current_version.py deleted file mode 100644 index 0c5e2a15..00000000 --- a/tests/more-tests/version/test_current_version.py +++ /dev/null @@ -1,38 +0,0 @@ -__pysd_version__ = "2.99.3" - -_namespace = {} -_dependencies = {} - -__data = {'scope': None, 'time': lambda: 0} - -_control_vars = { - "initial_time": lambda: 0, - "final_time": lambda: 20, - "time_step": lambda: 1, - "saveper": lambda: time_step() -} - - -def _init_outer_references(data): - for key in data: - __data[key] = data[key] - - -def time(): - return __data["time"]() - - -def initial_time(): - return __data["time"].initial_time() - - -def final_time(): - return __data["time"].final_time() - - -def time_step(): - return __data["time"].time_step() - - -def saveper(): - return __data["time"].saveper() diff --git a/tests/pytest.ini b/tests/pytest.ini index d6f5a6e0..982daecb 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -1,2 +1,2 @@ [pytest] -python_files = unit_test_*.py integration_test_*.py pytest_*/**/*.py +python_files = unit_test_*.py pytest_*/**/*.py pytest_*/*.py diff --git a/tests/pytest_builders/pytest_python.py b/tests/pytest_builders/pytest_python.py new file mode 100644 index 00000000..3a5b28bd --- /dev/null +++ b/tests/pytest_builders/pytest_python.py @@ -0,0 +1,129 @@ +import pytest +from pathlib import Path + +from pysd.builders.python.subscripts import SubscriptManager +from pysd.builders.python.python_model_builder import\ + ComponentBuilder, ElementBuilder, SectionBuilder +from pysd.builders.python.python_expressions_builder import\ + StructureBuilder, BuildAST +from pysd.translators.structures.abstract_model import\ + AbstractComponent, AbstractElement, AbstractSection, AbstractSubscriptRange + + +class TestStructureBuilder: + """ + Test for StructureBuilder + """ + + @pytest.fixture + def section(self): + return SectionBuilder( + AbstractSection( + "main", Path("here"), "__main__", + [], [], tuple(), tuple(), False, None + )) + + @pytest.fixture + def abstract_component(self): + return AbstractComponent([[], []], "") + + @pytest.fixture + def element(self, section, abstract_component): + return ElementBuilder( + AbstractElement("element", [abstract_component], "", None, ""), + section + ) + + @pytest.fixture + def component(self, element, section, abstract_component): + component_obj = ComponentBuilder(abstract_component, element, section) + component_obj.subscripts_dict = {} + return component_obj + + @pytest.fixture + def structure_builder(self, component): + return StructureBuilder(None, component) + + @pytest.mark.parametrize( + "arguments,expected", + [ + ( # 0 + {}, + {} + ), + ( # 1 + {"0": BuildAST("", {"a": 1}, {}, 0)}, + {"a": 1} + ), + ( # 2 + {"0": BuildAST("", {"a": 1}, {}, 0), + "1": BuildAST("", {"a": 1, "b": 3}, {}, 0)}, + {"a": 2, "b": 3} + ), + ( # 3 + {"0": BuildAST("", {"a": 1}, {}, 0), + "1": BuildAST("", {"a": 1, "b": 3, "c": 2}, {}, 0), + "2": BuildAST("", {"b": 5}, {}, 0)}, + {"a": 2, "b": 8, "c": 2} + ), + ], + ids=["0", "1", "2", "3"] + ) + def test_join_calls(self, structure_builder, arguments, expected): + assert structure_builder.join_calls(arguments) == expected + + @pytest.mark.parametrize( + "arguments,expected", + [ + ( # 0 + {}, + {} + ), + ( # 1 + {"0": BuildAST("", {}, {"a": [1]}, 0)}, + {"a": [1]} + ), + ( # 2a + {"0": BuildAST("", {}, {"a": [1]}, 0), + "1": BuildAST("", {}, {}, 0)}, + {"a": [1]} + ), + ( # 2b + {"0": BuildAST("", {}, {"a": [1]}, 0), + "1": BuildAST("", {}, {"b": [2, 3]}, 0)}, + {"a": [1], "b": [2, 3]} + ), + ( # 2c + {"0": BuildAST("", {}, {"a": [1]}, 0), + "1": BuildAST("", {}, {"b": [2, 3], "a": [1]}, 0)}, + {"a": [1], "b": [2, 3]} + ), + ( # 3 + {"0": BuildAST("", {}, {"a": [1]}, 0), + "1": BuildAST("", {}, {"b": [2, 3], "a": [1]}, 0), + "2": BuildAST("", {}, {"b": [2, 3], "c": [10]}, 0)}, + {"a": [1], "b": [2, 3], "c": [10]} + ), + ], + ids=["0", "1", "2a", "2b", "2c", "3"] + ) + def test__get_final_subscripts(self, structure_builder, + arguments, expected): + assert structure_builder.get_final_subscripts(arguments) == expected + + +class TestSubscriptManager: + @pytest.mark.parametrize( + "arguments,raise_type,error_message", + [ + ( # invalid definition + [[AbstractSubscriptRange("my subs", 5, [])], Path("here")], + ValueError, + "Invalid definition of subscript 'my subs':\n\t5" + ), + ], + ids=["invalid definition"] + ) + def test_invalid_subscripts(self, arguments, raise_type, error_message): + with pytest.raises(raise_type, match=error_message): + SubscriptManager(*arguments) diff --git a/tests/pytest_integration/pytest_integration_vensim_pathway.py b/tests/pytest_integration/pytest_integration_vensim_pathway.py new file mode 100644 index 00000000..f7dbc389 --- /dev/null +++ b/tests/pytest_integration/pytest_integration_vensim_pathway.py @@ -0,0 +1,583 @@ +import warnings +import shutil +import pytest + +from pysd.tools.benchmarking import runner, assert_frames_close + +# TODO add warnings catcher per test + +vensim_test = { + "abs": { + "folder": "abs", + "file": "test_abs.mdl" + }, + "active_initial": { + "folder": "active_initial", + "file": "test_active_initial.mdl" + }, + "active_initial_circular": { + "folder": "active_initial_circular", + "file": "test_active_initial_circular.mdl" + }, + "arithmetics": { + "folder": "arithmetics", + "file": "test_arithmetics.mdl" + }, + "arithmetics_exp": { + "folder": "arithmetics_exp", + "file": "test_arithmetics_exp.mdl" + }, + "arguments": { + "folder": "arguments", + "file": "test_arguments.mdl", + "rtol": 1e-2 # TODO test why it is failing with smaller tolerance + }, + "array_with_line_break": { + "folder": "array_with_line_break", + "file": "test_array_with_line_break.mdl" + }, + "builtin_max": { + "folder": "builtin_max", + "file": "builtin_max.mdl" + }, + "builtin_min": { + "folder": "builtin_min", + "file": "builtin_min.mdl" + }, + "chained_initialization": { + "folder": "chained_initialization", + "file": "test_chained_initialization.mdl" + }, + "conditional_subscripts": { + "folder": "conditional_subscripts", + "file": "test_conditional_subscripts.mdl" + }, + "constant_expressions": { + "folder": "constant_expressions", + "file": "test_constant_expressions.mdl" + }, + "control_vars": { + "folder": "control_vars", + "file": "test_control_vars.mdl" + }, + "data_from_other_model": { + "folder": "data_from_other_model", + "file": "test_data_from_other_model.mdl", + "data_files": "data.tab" + }, + "delay_fixed": { + "folder": "delay_fixed", + "file": "test_delay_fixed.mdl" + }, + "delay_numeric_error": { + "folder": "delay_numeric_error", + "file": "test_delay_numeric_error.mdl" + }, + "delay_parentheses": { + "folder": "delay_parentheses", + "file": "test_delay_parentheses.mdl" + }, + "delay_pipeline": { + "folder": "delay_pipeline", + "file": "test_pipeline_delays.mdl" + }, + "delays": { + "folder": "delays", + "file": "test_delays.mdl" + }, + "dynamic_final_time": { + "folder": "dynamic_final_time", + "file": "test_dynamic_final_time.mdl" + }, + "elm_count": { + "folder": "elm_count", + "file": "test_elm_count.mdl" + }, + "euler_step_vs_saveper": { + "folder": "euler_step_vs_saveper", + "file": "test_euler_step_vs_saveper.mdl" + }, + "except": { + "folder": "except", + "file": "test_except.mdl" + }, + "except_multiple": { + "folder": "except_multiple", + "file": "test_except_multiple.mdl" + }, + "exp": { + "folder": "exp", + "file": "test_exp.mdl" + }, + "exponentiation": { + "folder": "exponentiation", + "file": "exponentiation.mdl" + }, + "forecast": { + "folder": "forecast", + "file": "test_forecast.mdl" + }, + "function_capitalization": { + "folder": "function_capitalization", + "file": "test_function_capitalization.mdl" + }, + "fully_invalid_names": { + "folder": "fully_invalid_names", + "file": "test_fully_invalid_names.mdl" + }, + "game": { + "folder": "game", + "file": "test_game.mdl" + }, + "get_constants": pytest.param({ + "folder": "get_constants", + "file": "test_get_constants.mdl" + }, marks=pytest.mark.xfail(reason="csv files not implemented")), + "get_constants_subranges": { + "folder": "get_constants_subranges", + "file": "test_get_constants_subranges.mdl" + }, + "get_data": pytest.param({ + "folder": "get_data", + "file": "test_get_data.mdl" + }, marks=pytest.mark.xfail(reason="csv files not implemented")), + "get_data_args_3d_xls": { + "folder": "get_data_args_3d_xls", + "file": "test_get_data_args_3d_xls.mdl" + }, + "get_lookups_data_3d_xls": { + "folder": "get_lookups_data_3d_xls", + "file": "test_get_lookups_data_3d_xls.mdl" + }, + "get_lookups_subscripted_args": { + "folder": "get_lookups_subscripted_args", + "file": "test_get_lookups_subscripted_args.mdl" + }, + "get_lookups_subset": { + "folder": "get_lookups_subset", + "file": "test_get_lookups_subset.mdl" + }, + "get_mixed_definitions": { + "folder": "get_mixed_definitions", + "file": "test_get_mixed_definitions.mdl" + }, + "get_subscript_3d_arrays_xls": { + "folder": "get_subscript_3d_arrays_xls", + "file": "test_get_subscript_3d_arrays_xls.mdl" + }, + "get_with_missing_values_xlsx": { + "folder": "get_with_missing_values_xlsx", + "file": "test_get_with_missing_values_xlsx.mdl" + }, + "get_xls_cellrange": { + "folder": "get_xls_cellrange", + "file": "test_get_xls_cellrange.mdl" + }, + "if_stmt": { + "folder": "if_stmt", + "file": "if_stmt.mdl" + }, + "initial_function": { + "folder": "initial_function", + "file": "test_initial.mdl" + }, + "input_functions": { + "folder": "input_functions", + "file": "test_inputs.mdl" + }, + "invert_matrix": { + "folder": "invert_matrix", + "file": "test_invert_matrix.mdl" + }, + "limits": { + "folder": "limits", + "file": "test_limits.mdl" + }, + "line_breaks": { + "folder": "line_breaks", + "file": "test_line_breaks.mdl" + }, + "line_continuation": { + "folder": "line_continuation", + "file": "test_line_continuation.mdl" + }, + "ln": { + "folder": "ln", + "file": "test_ln.mdl" + }, + "log": { + "folder": "log", + "file": "test_log.mdl" + }, + "logicals": { + "folder": "logicals", + "file": "test_logicals.mdl" + }, + "lookups": { + "folder": "lookups", + "file": "test_lookups.mdl" + }, + "lookups_funcnames": { + "folder": "lookups_funcnames", + "file": "test_lookups_funcnames.mdl" + }, + "lookups_inline": { + "folder": "lookups_inline", + "file": "test_lookups_inline.mdl" + }, + "lookups_inline_bounded": { + "folder": "lookups_inline_bounded", + "file": "test_lookups_inline_bounded.mdl" + }, + "lookups_inline_spaces": { + "folder": "lookups_inline_spaces", + "file": "test_lookups_inline_spaces.mdl" + }, + "lookups_with_expr": { + "folder": "lookups_with_expr", + "file": "test_lookups_with_expr.mdl" + }, + "lookups_without_range": { + "folder": "lookups_without_range", + "file": "test_lookups_without_range.mdl" + }, + "macro_cross_reference": { + "folder": "macro_cross_reference", + "file": "test_macro_cross_reference.mdl" + }, + "macro_expression": { + "folder": "macro_expression", + "file": "test_macro_expression.mdl" + }, + "macro_multi_expression": { + "folder": "macro_multi_expression", + "file": "test_macro_multi_expression.mdl" + }, + "macro_multi_macros": { + "folder": "macro_multi_macros", + "file": "test_macro_multi_macros.mdl" + }, + "macro_stock": { + "folder": "macro_stock", + "file": "test_macro_stock.mdl" + }, + "macro_trailing_definition": { + "folder": "macro_trailing_definition", + "file": "test_macro_trailing_definition.mdl" + }, + "model_doc": { + "folder": "model_doc", + "file": "model_doc.mdl" + }, + "multiple_lines_def": { + "folder": "multiple_lines_def", + "file": "test_multiple_lines_def.mdl" + }, + "na": { + "folder": "na", + "file": "test_na.mdl" + }, + "nested_functions": { + "folder": "nested_functions", + "file": "test_nested_functions.mdl" + }, + "number_handling": { + "folder": "number_handling", + "file": "test_number_handling.mdl" + }, + "odd_number_quotes": { + "folder": "odd_number_quotes", + "file": "teacup_3quotes.mdl" + }, + "parentheses": { + "folder": "parentheses", + "file": "test_parens.mdl" + }, + "reference_capitalization": { + "folder": "reference_capitalization", + "file": "test_reference_capitalization.mdl" + }, + "repeated_subscript": { + "folder": "repeated_subscript", + "file": "test_repeated_subscript.mdl" + }, + "rounding": { + "folder": "rounding", + "file": "test_rounding.mdl" + }, + "sample_if_true": { + "folder": "sample_if_true", + "file": "test_sample_if_true.mdl" + }, + "smaller_range": { + "folder": "smaller_range", + "file": "test_smaller_range.mdl" + }, + "smooth": { + "folder": "smooth", + "file": "test_smooth.mdl" + }, + "smooth_and_stock": { + "folder": "smooth_and_stock", + "file": "test_smooth_and_stock.mdl" + }, + "special_characters": { + "folder": "special_characters", + "file": "test_special_variable_names.mdl" + }, + "sqrt": { + "folder": "sqrt", + "file": "test_sqrt.mdl" + }, + "subrange_merge": { + "folder": "subrange_merge", + "file": "test_subrange_merge.mdl" + }, + "subscript_1d_arrays": { + "folder": "subscript_1d_arrays", + "file": "test_subscript_1d_arrays.mdl" + }, + "subscript_2d_arrays": { + "folder": "subscript_2d_arrays", + "file": "test_subscript_2d_arrays.mdl" + }, + "subscript_3d_arrays": { + "folder": "subscript_3d_arrays", + "file": "test_subscript_3d_arrays.mdl" + }, + "subscript_3d_arrays_lengthwise": { + "folder": "subscript_3d_arrays_lengthwise", + "file": "test_subscript_3d_arrays_lengthwise.mdl" + }, + "subscript_3d_arrays_widthwise": { + "folder": "subscript_3d_arrays_widthwise", + "file": "test_subscript_3d_arrays_widthwise.mdl" + }, + "subscript_aggregation": { + "folder": "subscript_aggregation", + "file": "test_subscript_aggregation.mdl" + }, + "subscript_constant_call": { + "folder": "subscript_constant_call", + "file": "test_subscript_constant_call.mdl" + }, + "subscript_copy": { + "folder": "subscript_copy", + "file": "test_subscript_copy.mdl" + }, + "subscript_copy2": { + "folder": "subscript_copy", + "file": "test_subscript_copy2.mdl" + }, + "subscript_docs": { + "folder": "subscript_docs", + "file": "subscript_docs.mdl" + }, + "subscript_element_name": { + "folder": "subscript_element_name", + "file": "test_subscript_element_name.mdl" + }, + "subscript_individually_defined_1_of_2d_arrays": { + "folder": "subscript_individually_defined_1_of_2d_arrays", + "file": "subscript_individually_defined_1_of_2d_arrays.mdl" + }, + "subscript_individually_defined_1_of_2d_arrays_from_floats": { + "folder": "subscript_individually_defined_1_of_2d_arrays_from_floats", + "file": "subscript_individually_defined_1_of_2d_arrays_from_floats.mdl" + }, + "subscript_individually_defined_1d_arrays": { + "folder": "subscript_individually_defined_1d_arrays", + "file": "subscript_individually_defined_1d_arrays.mdl" + }, + "subscript_individually_defined_stocks": { + "folder": "subscript_individually_defined_stocks", + "file": "test_subscript_individually_defined_stocks.mdl" + }, + "subscript_logicals": { + "folder": "subscript_logicals", + "file": "test_subscript_logicals.mdl" + }, + "subscript_mapping_simple": { + "folder": "subscript_mapping_simple", + "file": "test_subscript_mapping_simple.mdl" + }, + "subscript_mapping_vensim": { + "folder": "subscript_mapping_vensim", + "file": "test_subscript_mapping_vensim.mdl" + }, + "subscript_mixed_assembly": { + "folder": "subscript_mixed_assembly", + "file": "test_subscript_mixed_assembly.mdl" + }, + "subscript_multiples": { + "folder": "subscript_multiples", + "file": "test_multiple_subscripts.mdl" + }, + "subscript_numeric_range": { + "folder": "subscript_numeric_range", + "file": "test_subscript_numeric_range.mdl" + }, + "subscript_selection": { + "folder": "subscript_selection", + "file": "subscript_selection.mdl" + }, + "subscript_subranges": { + "folder": "subscript_subranges", + "file": "test_subscript_subrange.mdl" + }, + "subscript_subranges_equal": { + "folder": "subscript_subranges_equal", + "file": "test_subscript_subrange_equal.mdl" + }, + "subscript_switching": { + "folder": "subscript_switching", + "file": "subscript_switching.mdl" + }, + "subscript_transposition": { + "folder": "subscript_transposition", + "file": "test_subscript_transposition.mdl" + }, + "subscript_updimensioning": { + "folder": "subscript_updimensioning", + "file": "test_subscript_updimensioning.mdl" + }, + "subscripted_delays": { + "folder": "subscripted_delays", + "file": "test_subscripted_delays.mdl" + }, + "subscripted_flows": { + "folder": "subscripted_flows", + "file": "test_subscripted_flows.mdl" + }, + "subscripted_if_then_else": { + "folder": "subscripted_if_then_else", + "file": "test_subscripted_if_then_else.mdl" + }, + "subscripted_logicals": { + "folder": "subscripted_logicals", + "file": "test_subscripted_logicals.mdl" + }, + "subscripted_lookups": { + "folder": "subscripted_lookups", + "file": "test_subscripted_lookups.mdl" + }, + "subscripted_round": { + "folder": "subscripted_round", + "file": "test_subscripted_round.mdl" + }, + "subscripted_smooth": { + "folder": "subscripted_smooth", + "file": "test_subscripted_smooth.mdl" + }, + "subscripted_trend": { + "folder": "subscripted_trend", + "file": "test_subscripted_trend.mdl" + }, + "subscripted_trig": { + "folder": "subscripted_trig", + "file": "test_subscripted_trig.mdl" + }, + "subscripted_xidz": { + "folder": "subscripted_xidz", + "file": "test_subscripted_xidz.mdl" + }, + "subset_duplicated_coord": { + "folder": "subset_duplicated_coord", + "file": "test_subset_duplicated_coord.mdl" + }, + "time": { + "folder": "time", + "file": "test_time.mdl" + }, + "trend": { + "folder": "trend", + "file": "test_trend.mdl" + }, + "trig": { + "folder": "trig", + "file": "test_trig.mdl" + }, + "unchangeable_constant": { + "folder": "unchangeable_constant", + "file": "test_unchangeable_constant.mdl" + }, + "unicode_characters": { + "folder": "unicode_characters", + "file": "unicode_test_model.mdl" + }, + "variable_ranges": { + "folder": "variable_ranges", + "file": "test_variable_ranges.mdl" + }, + "xidz_zidz": { + "folder": "xidz_zidz", + "file": "xidz_zidz.mdl" + }, + "zeroled_decimals": { + "folder": "zeroled_decimals", + "file": "test_zeroled_decimals.mdl" + } +} + + +@pytest.mark.parametrize( + "test_data", + [item for item in vensim_test.values()], + ids=list(vensim_test) +) +class TestIntegrateVensim: + """ + Test for splitting Vensim views in modules and submodules + """ + @pytest.fixture + def test_folder(self, tmp_path, _test_models, test_data): + """ + Copy test folder to a temporary folder therefore we avoid creating + PySD model files in the original folder + """ + test_folder = tmp_path.joinpath(test_data["folder"]) + shutil.copytree( + _test_models.joinpath(test_data["folder"]), + test_folder + ) + return test_folder + + @pytest.fixture + def model_path(self, test_folder, test_data): + """Return model path""" + return test_folder.joinpath(test_data["file"]) + + @pytest.fixture + def data_path(self, test_folder, test_data): + """Fixture for models with data_path""" + if "data_files" in test_data: + if isinstance(test_data["data_files"], str): + return test_folder.joinpath(test_data["data_files"]) + elif isinstance(test_data["data_files"], list): + return [ + test_folder.joinpath(file) + for file in test_data["data_files"] + ] + else: + return { + test_folder.joinpath(file): values + for file, values in test_data["data_files"].items() + } + else: + return None + + @pytest.fixture + def kwargs(self, test_data): + """Fixture for atol and rtol""" + kwargs = {} + if "atol" in test_data: + kwargs["atol"] = test_data["atol"] + if "rtol" in test_data: + kwargs["rtol"] = test_data["rtol"] + return kwargs + + def test_read_vensim_file(self, model_path, data_path, kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + output, canon = runner(model_path, data_files=data_path) + assert_frames_close(output, canon, **kwargs) diff --git a/tests/pytest_integration/pytest_integration_xmile_pathway.py b/tests/pytest_integration/pytest_integration_xmile_pathway.py new file mode 100644 index 00000000..8772c5f6 --- /dev/null +++ b/tests/pytest_integration/pytest_integration_xmile_pathway.py @@ -0,0 +1,263 @@ +import warnings +import shutil +import pytest + +from pysd.tools.benchmarking import runner, assert_frames_close + +# TODO add warnings catcher per test + +xmile_test = { + "abs": { + "folder": "abs", + "file": "test_abs.xmile" + }, + "active_initial": pytest.param({ + "folder": "active_initial", + "file": "test_active_initial.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "arithmetics_exp": { + "folder": "arithmetics_exp", + "file": "test_arithmetics_exp.xmile" + }, + "builtin_max": { + "folder": "builtin_max", + "file": "builtin_max.xmile" + }, + "builtin_min": { + "folder": "builtin_min", + "file": "builtin_min.xmile" + }, + "chained_initialization": { + "folder": "chained_initialization", + "file": "test_chained_initialization.xmile" + }, + "comparisons": { + "folder": "comparisons", + "file": "comparisons.xmile" + }, + "constant_expressions": { + "folder": "constant_expressions", + "file": "test_constant_expressions.xmile" + }, + "eval_order": { + "folder": "eval_order", + "file": "eval_order.xmile" + }, + "exp": { + "folder": "exp", + "file": "test_exp.xmile" + }, + "exponentiation": { + "folder": "exponentiation", + "file": "exponentiation.xmile" + }, + "function_capitalization": { + "folder": "function_capitalization", + "file": "test_function_capitalization.xmile" + }, + "game": { + "folder": "game", + "file": "test_game.xmile" + }, + "if_stmt": { + "folder": "if_stmt", + "file": "if_stmt.xmile" + }, + "initial_function": { + "folder": "initial_function", + "file": "test_initial.xmile" + }, + "limits": { + "folder": "limits", + "file": "test_limits.xmile" + }, + "line_breaks": { + "folder": "line_breaks", + "file": "test_line_breaks.xmile" + }, + "line_continuation": { + "folder": "line_continuation", + "file": "test_line_continuation.xmile" + }, + "ln": { + "folder": "ln", + "file": "test_ln.xmile" + }, + "log": { + "folder": "log", + "file": "test_log.xmile" + }, + "logicals": { + "folder": "logicals", + "file": "test_logicals.xmile" + }, + "lookups": { + "folder": "lookups", + "file": "test_lookups.xmile" + }, + "lookups_no-indirect": pytest.param({ + "folder": "lookups", + "file": "test_lookups_no-indirect.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "lookups_xpts_sep": { + "folder": "lookups", + "file": "test_lookups_xpts_sep.xmile" + }, + "lookups_xscale": { + "folder": "lookups", + "file": "test_lookups_xscale.xmile" + }, + "lookups_ypts_sep": { + "folder": "lookups", + "file": "test_lookups_ypts_sep.xmile" + }, + "lookups_inline": { + "folder": "lookups_inline", + "file": "test_lookups_inline.xmile" + }, + "macro_expression": pytest.param({ + "folder": "macro_expression", + "file": "test_macro_expression.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "macro_multi_expression": pytest.param({ + "folder": "macro_multi_expression", + "file": "test_macro_multi_expression.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "macro_multi_macros": pytest.param({ + "folder": "macro_multi_macros", + "file": "test_macro_multi_macros.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "macro_stock": pytest.param({ + "folder": "macro_stock", + "file": "test_macro_stock.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "model_doc": { + "folder": "model_doc", + "file": "model_doc.xmile" + }, + "number_handling": { + "folder": "number_handling", + "file": "test_number_handling.xmile" + }, + "parentheses": { + "folder": "parentheses", + "file": "test_parens.xmile" + }, + "reference_capitalization": { + "folder": "reference_capitalization", + "file": "test_reference_capitalization.xmile" + }, + "rounding": { + "folder": "rounding", + "file": "test_rounding.xmile" + }, + "smooth_and_stock": pytest.param({ + "folder": "smooth_and_stock", + "file": "test_smooth_and_stock.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "special_characters": pytest.param({ + "folder": "special_characters", + "file": "test_special_variable_names.xmile" + }, marks=pytest.mark.xfail(reason="failing originally")), + "sqrt": { + "folder": "sqrt", + "file": "test_sqrt.xmile" + }, + "subscript_1d_arrays": pytest.param({ + "folder": "subscript_1d_arrays", + "file": "test_subscript_1d_arrays.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_constant_call": pytest.param({ + "folder": "subscript_constant_call", + "file": "test_subscript_constant_call.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_individually_defined_1d_arrays": { + "folder": "subscript_individually_defined_1d_arrays", + "file": "subscript_individually_defined_1d_arrays.xmile" + }, + "subscript_mixed_assembly": pytest.param({ + "folder": "subscript_mixed_assembly", + "file": "test_subscript_mixed_assembly.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_multiples": pytest.param({ + "folder": "subscript_multiples", + "file": "test_multiple_subscripts.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_subranges": pytest.param({ + "folder": "subscript_subranges", + "file": "test_subscript_subrange.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_subranges_equal": pytest.param({ + "folder": "subscript_subranges_equal", + "file": "test_subscript_subrange_equal.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscript_updimensioning": pytest.param({ + "folder": "subscript_updimensioning", + "file": "test_subscript_updimensioning.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscripted_flows": pytest.param({ + "folder": "subscripted_flows", + "file": "test_subscripted_flows.xmile" + }, marks=pytest.mark.xfail(reason="eqn with ??? in the model")), + "subscripted_trig": { + "folder": "subscripted_trig", + "file": "test_subscripted_trig.xmile" + }, + "trig": { + "folder": "trig", + "file": "test_trig.xmile" + }, + "xidz_zidz": { + "folder": "xidz_zidz", + "file": "xidz_zidz.xmile" + }, + "zeroled_decimals": { + "folder": "zeroled_decimals", + "file": "test_zeroled_decimals.xmile" + } +} + + +@pytest.mark.parametrize( + "test_data", + [item for item in xmile_test.values()], + ids=list(xmile_test) +) +class TestIntegrateXmile: + """ + Test for full translation and integration of models + """ + + @pytest.fixture + def test_folder(self, tmp_path, _test_models, test_data): + """ + Copy test folder to a temporary folder therefore we avoid creating + PySD model files in the original folder + """ + test_folder = tmp_path.joinpath(test_data["folder"]) + shutil.copytree( + _test_models.joinpath(test_data["folder"]), + test_folder + ) + return test_folder + + @pytest.fixture + def model_path(self, test_folder, test_data): + """Return model path""" + return test_folder.joinpath(test_data["file"]) + + @pytest.fixture + def kwargs(self, test_data): + """Fixture for atol and rtol""" + kwargs = {} + if "atol" in test_data: + kwargs["atol"] = test_data["atol"] + if "rtol" in test_data: + kwargs["rtol"] = test_data["rtol"] + return kwargs + + def test_read_vensim_file(self, model_path, kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + output, canon = runner(model_path) + assert_frames_close(output, canon, **kwargs) diff --git a/tests/pytest_pysd/pytest_errors.py b/tests/pytest_pysd/pytest_errors.py new file mode 100644 index 00000000..58f4d79a --- /dev/null +++ b/tests/pytest_pysd/pytest_errors.py @@ -0,0 +1,110 @@ +import pytest +import shutil + +from pysd import read_vensim, read_xmile, load + + +@pytest.fixture +def original_path(_root, name, suffix): + return _root / "more-tests" / name / f"test_{name}.{suffix}" + + +@pytest.fixture +def model_path(original_path, shared_tmpdir, _root): + """ + Copy test folder to a temporary folder therefore we avoid creating + PySD model files in the original folder + """ + new_file = shared_tmpdir / original_path.name + shutil.copy( + original_path, + new_file + ) + return new_file + + +@pytest.mark.parametrize( + "name,suffix,loader,raise_type,error_message", + [ + ( # load_old_version + "old_version", + "py", + load, + ImportError, + r"Not able to import the model\. The model was translated " + r"with a not compatible version of PySD:\n\tPySD 1\.5\.0" + ), + ( # load_type + "type_error", + "py", + load, + ImportError, + r".*Not able to import the model\. This may be because the " + "model was compiled with an earlier version of PySD, you can " + r"check on the top of the model file you are trying to load\..*" + ), + ( # not_vensim_model + "not_vensim", + "txt", + read_vensim, + ValueError, + "The file to translate, " + "'.*test_not_vensim.txt' is not a " + r"Vensim model\. It must end with \.mdl extension\." + ), + ( # not_xmile_model + "not_vensim", + "txt", + read_xmile, + ValueError, + "The file to translate, " + "'.*test_not_vensim.txt' is not a " + r"Xmile model\. It must end with any of \.xmile, \.xml, " + r"\.stmx extensions\." + ), + ( # circular_reference + "circular_reference", + "py", + load, + ValueError, + r"Circular initialization\.\.\.\nNot able to initialize the " + "following objects:\n\t_integ_integ\n\t_delay_delay" + ), + ], + ids=[ + "old_version", "load_type", + "not_vensim_model", "not_xmile_model", + "circular_reference" + ] +) +def test_loading_error(loader, model_path, raise_type, error_message): + with pytest.raises(raise_type, match=error_message): + loader(model_path) + + +@pytest.mark.parametrize( + "name,suffix", + [ + ( # load_old_version + "not_implemented_and_incomplete", + "mdl", + ), + ] +) +def test_not_implemented_and_incomplete(model_path): + with pytest.warns(UserWarning) as ws: + model = read_vensim(model_path) + assert "'incomplete var' has no equation specified"\ + in str(ws[0].message) + assert "Trying to translate 'MY FUNC' which it is not implemented"\ + " on PySD. The translated model will crash..."\ + in str(ws[1].message) + + with pytest.warns(RuntimeWarning, + match="Call to undefined function, calling dependencies " + "and returning NaN"): + model["incomplete var"] + + with pytest.raises(NotImplementedError, + match="Not implemented function 'my_func'"): + model["not implemented function"] diff --git a/tests/pytest_pysd/pytest_functions.py b/tests/pytest_pysd/pytest_functions.py new file mode 100644 index 00000000..c3ecb58f --- /dev/null +++ b/tests/pytest_pysd/pytest_functions.py @@ -0,0 +1,350 @@ +import pytest +import numpy as np +import xarray as xr + +from pysd.py_backend.components import Time +from pysd.py_backend.functions import\ + ramp, step, pulse, xidz, zidz, if_then_else, sum, prod, vmin, vmax,\ + invert_matrix + + +class TestInputFunctions(): + def test_ramp(self): + assert ramp(lambda: 14, .5, 10, 18) == 2 + + assert ramp(lambda: 4, .5, 10, 18) == 0 + + assert ramp(lambda: 24, .5, 10, 18) == 4 + + assert ramp(lambda: 24, .5, 10) == 7 + + assert ramp(lambda: 50, .5, 10) == 20 + + def test_step(self): + assert step(lambda: 5, 1, 10) == 0 + + assert step(lambda: 15, 1, 10) == 1 + + assert step(lambda: 10, 1, 10) == 1 + + def test_pulse(self): + assert pulse(lambda: 0, 1, width=3) == 0 + + assert pulse(lambda: 1, 1, width=3) == 1 + + assert pulse(lambda: 2, 1, width=3) == 1 + + assert pulse(lambda: 4, 1, width=3) == 0 + + assert pulse(lambda: 5, 1, width=3) == 0 + + def test_pulse_chain(self): + # before train starts + assert pulse(lambda: 0, 1, repeat_time=5, width=3, end=12) == 0 + # on train start + assert pulse(lambda: 1, 1, repeat_time=5, width=3, end=12) == 1 + # within first pulse + assert pulse(lambda: 2, 1, repeat_time=5, width=3, end=12) == 1 + # end of first pulse + assert pulse(lambda: 4, 1, repeat_time=5, width=3, end=12) == 0 + # after first pulse before second + assert pulse(lambda: 5, 1, repeat_time=5, width=3, end=12) == 0 + # on start of second pulse + assert pulse(lambda: 6, 1, repeat_time=5, width=3, end=12) == 1 + # within second pulse + assert pulse(lambda: 7, 1, repeat_time=5, width=3, end=12) == 1 + # after second pulse + assert pulse(lambda: 10, 1, repeat_time=5, width=3, end=12) == 0 + # on third pulse + assert pulse(lambda: 11, 1, repeat_time=5, width=3, end=12) == 1 + # on train end + assert pulse(lambda: 12, 1, repeat_time=5, width=3, end=12) == 0 + # after train + assert pulse(lambda: 15, 1, repeat_time=5, width=3, end=13) == 0 + + def test_pulse_magnitude(self): + # Pulse function with repeat time + # before first impulse + t = Time() + t.set_control_vars(initial_time=0, time_step=1) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 0 + # first impulse + t.update(2) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 10 + # after first impulse and before second + t.update(4) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 0 + # second impulse + t.update(7) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 10 + # after second and before third impulse + t.update(9) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 0 + # third impulse + t.update(12) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 10 + # after third impulse + t.update(14) + assert pulse(t, 2, repeat_time=5, magnitude=10) == 0 + + t = Time() + t.set_control_vars(initial_time=0, time_step=0.2) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 0 + # first impulse + t.update(2) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 0 + # after first impulse and before second + t.update(3) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 25 + # second impulse + t.update(7) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 0 + # after second and before third impulse + t.update(8) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 25 + # third impulse + t.update(12) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 0 + # after third impulse + t.update(13) + assert pulse(t, 3, repeat_time=5, magnitude=5) == 25 + + # Pulse function without repeat time + # before first impulse + t = Time() + t.set_control_vars(initial_time=0, time_step=1) + assert pulse(t, 2, magnitude=10) == 0 + # first impulse + t.update(2) + assert pulse(t, 2, magnitude=10) == 10 + # after first impulse and before second + t.update(4) + assert pulse(t, 2, magnitude=10) == 0 + # second impulse + t.update(7) + assert pulse(t, 2, magnitude=10) == 0 + # after second and before third impulse + t.update(9) + assert pulse(t, 2, magnitude=10) == 0 + + t = Time() + t.set_control_vars(initial_time=0, time_step=0.1) + assert pulse(t, 4, magnitude=10) == 0 + # first impulse + t.update(2) + assert pulse(t, 4, magnitude=10) == 0 + # after first impulse and before second + t.update(4) + assert pulse(t, 4, magnitude=10) == 100 + # second impulse + t.update(7) + assert pulse(t, 4, magnitude=10) == 0 + # after second and before third impulse + t.update(9) + assert pulse(t, 4, magnitude=10) == 0 + + def test_numeric_error(self): + time = Time() + time.set_control_vars(initial_time=0, time_step=0.1, final_time=10) + err = 4e-16 + + # upper numeric error + time.update(3 + err) + assert 3 != time(), "there is no numeric error included" + + assert pulse(time, 3) == 1 + assert pulse(time, 1, repeat_time=2) == 1 + + # lower numeric error + time.update(3 - err) + assert 3 != time(), "there is no numeric error included" + + assert pulse(time, 3) == 1 + assert pulse(time, 1, repeat_time=2) == 1 + + def test_xidz(self): + assert xidz(1, -0.00000001, 5) == 5 + assert xidz(1, 0, 5) == 5 + assert xidz(1, 8, 5) == 0.125 + + def test_zidz(self): + assert zidz(1, -0.00000001) == 0 + assert zidz(1, 0) == 0 + assert zidz(1, 8) == 0.125 + + +class TestLogicFunctions(): + def test_if_then_else_basic(self): + assert if_then_else(True, lambda: 1, lambda: 0) == 1 + assert if_then_else(False, lambda: 1, lambda: 0) == 0 + + # Ensure lazzy evaluation + assert if_then_else(True, lambda: 1, lambda: 1/0) == 1 + assert if_then_else(False, lambda: 1/0, lambda: 0) == 0 + + with pytest.raises(ZeroDivisionError): + if_then_else(True, lambda: 1/0, lambda: 0) + with pytest.raises(ZeroDivisionError): + if_then_else(False, lambda: 1, lambda: 1/0) + + def test_if_then_else_with_subscripted(self): + # this test only test the lazzy evaluation and basics + # subscripted_if_then_else test all the possibilities + coords = {'dim1': [0, 1], 'dim2': [0, 1]} + dims = list(coords) + + xr_true = xr.DataArray([[True, True], [True, True]], coords, dims) + xr_false = xr.DataArray([[False, False], [False, False]], coords, dims) + xr_mixed = xr.DataArray([[True, False], [False, True]], coords, dims) + + out_mixed = xr.DataArray([[1, 0], [0, 1]], coords, dims) + + assert if_then_else(xr_true, lambda: 1, lambda: 0) == 1 + assert if_then_else(xr_false, lambda: 1, lambda: 0) == 0 + assert if_then_else(xr_mixed, lambda: 1, lambda: 0).equals(out_mixed) + + # Ensure lazzy evaluation + assert if_then_else(xr_true, lambda: 1, lambda: 1/0) == 1 + assert if_then_else(xr_false, lambda: 1/0, lambda: 0) == 0 + + with pytest.raises(ZeroDivisionError): + if_then_else(xr_true, lambda: 1/0, lambda: 0) + with pytest.raises(ZeroDivisionError): + if_then_else(xr_false, lambda: 1, lambda: 1/0) + with pytest.raises(ZeroDivisionError): + if_then_else(xr_mixed, lambda: 1/0, lambda: 0) + with pytest.raises(ZeroDivisionError): + if_then_else(xr_mixed, lambda: 1, lambda: 1/0) + + +class TestFunctions(): + + def test_sum(self): + """ + Test for sum function + """ + coords = {'d1': [9, 1], 'd2': [2, 4]} + coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} + dims = ['d1', 'd2'] + + data = xr.DataArray([[1, 2], [3, 4]], coords, dims) + + assert sum(data, dim=['d1']).equals( + xr.DataArray([4, 6], coords_d2, ['d2'])) + assert sum(data, dim=['d2']).equals( + xr.DataArray([3, 7], coords_d1, ['d1'])) + assert sum(data, dim=['d1', 'd2']) == 10 + assert sum(data) == 10 + + def test_prod(self): + """ + Test for sum function + """ + coords = {'d1': [9, 1], 'd2': [2, 4]} + coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} + dims = ['d1', 'd2'] + + data = xr.DataArray([[1, 2], [3, 4]], coords, dims) + + assert prod(data, dim=['d1']).equals( + xr.DataArray([3, 8], coords_d2, ['d2'])) + assert prod(data, dim=['d2']).equals( + xr.DataArray([2, 12], coords_d1, ['d1'])) + assert prod(data, dim=['d1', 'd2']) == 24 + assert prod(data) == 24 + + def test_vmin(self): + """ + Test for vmin function + """ + coords = {'d1': [9, 1], 'd2': [2, 4]} + coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} + dims = ['d1', 'd2'] + + data = xr.DataArray([[1, 2], [3, 4]], coords, dims) + + assert vmin(data, dim=['d1']).equals( + xr.DataArray([1, 2], coords_d2, ['d2'])) + assert vmin(data, dim=['d2']).equals( + xr.DataArray([1, 3], coords_d1, ['d1'])) + assert vmin(data, dim=['d1', 'd2']) == 1 + assert vmin(data) == 1 + + def test_vmax(self): + """ + Test for vmax function + """ + coords = {'d1': [9, 1], 'd2': [2, 4]} + coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} + dims = ['d1', 'd2'] + + data = xr.DataArray([[1, 2], [3, 4]], coords, dims) + + assert vmax(data, dim=['d1']).equals( + xr.DataArray([3, 4], coords_d2, ['d2'])) + assert vmax(data, dim=['d2']).equals( + xr.DataArray([2, 4], coords_d1, ['d1'])) + assert vmax(data, dim=['d1', 'd2']) == 4 + assert vmax(data) == 4 + + def test_invert_matrix(self): + """ + Test for invert_matrix function + """ + coords1 = {'d1': ['a', 'b'], 'd2': ['a', 'b']} + coords2 = {'d0': ['1', '2'], 'd1': ['a', 'b'], 'd2': ['a', 'b']} + coords3 = {'d0': ['1', '2'], + 'd1': ['a', 'b', 'c'], + 'd2': ['a', 'b', 'c']} + + data1 = xr.DataArray([[1, 2], [3, 4]], coords1, ['d1', 'd2']) + data2 = xr.DataArray([[[1, 2], [3, 4]], [[-1, 2], [5, 4]]], + coords2, + ['d0', 'd1', 'd2']) + data3 = xr.DataArray([[[1, 2, 3], [3, 7, 2], [3, 4, 6]], + [[-1, 2, 3], [4, 7, 3], [5, 4, 6]]], + coords3, + ['d0', 'd1', 'd2']) + + for data in [data1, data2, data3]: + datai = invert_matrix(data) + assert data.dims == datai.dims + + if len(data.shape) == 2: + # two dimensions xarrays + assert ( + abs(np.dot(data, datai) - np.dot(datai, data)) + < 1e-14 + ).all() + assert ( + abs(np.dot(data, datai) - np.identity(data.shape[-1])) + < 1e-14 + ).all() + else: + # three dimensions xarrays + for i in range(data.shape[0]): + assert ( + abs(np.dot(data[i], datai[i]) + - np.dot(datai[i], data[i])) + < 1e-14 + ).all() + assert ( + abs(np.dot(data[i], datai[i]) + - np.identity(data.shape[-1])) + < 1e-14 + ).all() + + def test_incomplete(self): + from pysd.py_backend.functions import incomplete + from warnings import catch_warnings + + with catch_warnings(record=True) as w: + incomplete() + assert len(w) == 1 + assert 'Call to undefined function' in str(w[-1].message) + + def test_not_implemented_function(self): + from pysd.py_backend.functions import not_implemented_function + + with pytest.raises(NotImplementedError): + not_implemented_function("NIF") diff --git a/tests/pytest_pysd/pytest_model_attributes.py b/tests/pytest_pysd/pytest_model_attributes.py new file mode 100644 index 00000000..e48935bc --- /dev/null +++ b/tests/pytest_pysd/pytest_model_attributes.py @@ -0,0 +1,88 @@ +import shutil +import pytest +from pathlib import Path + +import pysd + + +@pytest.fixture(scope="class") +def input_file(shared_tmpdir, _root): + input_path = Path("more-tests/split_model/input.xlsx") + shutil.copy( + _root.joinpath(input_path), + shared_tmpdir.joinpath(input_path.name) + ) + + +@pytest.mark.parametrize( + "model_path,subview_sep", + [ + ( # teacup + Path("test-models/samples/teacup/teacup.mdl"), + None + ), + ( # split_views + Path("more-tests/split_model/test_split_model.mdl"), + [] + ), + ( # split_subviews + Path("more-tests/split_model/test_split_model_subviews.mdl"), + ["."] + ), + ( # split_sub_subviews + Path("more-tests/split_model/test_split_model_sub_subviews.mdl"), + [".", "-"] + ) + ], + ids=["teacup", "split_views", "split_subviews", "split_sub_subviews"] +) +class TestModelProperties(): + + @pytest.fixture + def model(self, shared_tmpdir, model_path, subview_sep, _root, input_file): + """ + Translate the model or read a translated version. + This way each file is only translated once. + """ + # expected file + file = shared_tmpdir.joinpath(model_path.with_suffix(".py").name) + if file.is_file(): + # load already translated file + return pysd.load(file) + else: + # copy mdl file to tmp_dir and translate it + file = shared_tmpdir.joinpath(model_path.name) + shutil.copy(_root.joinpath(model_path), file) + return pysd.read_vensim( + file, + split_views=(subview_sep is not None), subview_sep=subview_sep) + + def test_propierties(self, model): + # test are equal to model attributes they are copying + assert model.namespace == model._namespace + assert model.subscripts == model._subscript_dict + assert model.dependencies == model._dependencies + if model._modules: + assert model.modules == model._modules + else: + assert model.modules is None + + # test thatwhen modifying a propierty by the user the model + # attribute remains the same + ns = model.namespace + ns["Time"] = "my_new_time" + assert ns != model._namespace + assert ns != model.namespace + sd = model.subscripts + sd["my_new_subs"] = ["s1", "s2", "s3"] + assert sd != model._subscript_dict + assert sd != model.subscripts + ds = model.dependencies + ds["my_var"] = {"time": 1} + assert ds != model._dependencies + assert ds != model.dependencies + if model._modules: + ms = model.modules + del ms[list(ms)[0]] + assert ms != model._modules + assert ms != model.modules diff --git a/tests/pytest_pysd/pytest_random.py b/tests/pytest_pysd/pytest_random.py new file mode 100644 index 00000000..7e6e1b1d --- /dev/null +++ b/tests/pytest_pysd/pytest_random.py @@ -0,0 +1,31 @@ + +import pytest +import shutil + +import pysd + + +class TestRandomModel: + """Submodel selecting class""" + # messages for selecting submodules + @pytest.fixture(scope="class") + def model_path(self, shared_tmpdir, _root): + """ + Copy test folder to a temporary folder therefore we avoid creating + PySD model files in the original folder + """ + new_file = shared_tmpdir.joinpath("test_random.mdl") + shutil.copy( + _root.joinpath("more-tests/random/test_random.mdl"), + new_file + ) + return new_file + + def test_translate(self, model_path): + """ + Translate the model or read a translated version. + This way each file is only translated once. + """ + # expected file + model = pysd.read_vensim(model_path) + model.run() diff --git a/tests/pytest_pysd/user_interaction/pytest_select_submodel.py b/tests/pytest_pysd/pytest_select_submodel.py similarity index 81% rename from tests/pytest_pysd/user_interaction/pytest_select_submodel.py rename to tests/pytest_pysd/pytest_select_submodel.py index 0e101fe0..c4759aea 100644 --- a/tests/pytest_pysd/user_interaction/pytest_select_submodel.py +++ b/tests/pytest_pysd/pytest_select_submodel.py @@ -7,6 +7,15 @@ import pysd +@pytest.fixture(scope="class") +def input_file(shared_tmpdir, _root): + input_path = Path("more-tests/split_model/input.xlsx") + shutil.copy( + _root.joinpath(input_path), + shared_tmpdir.joinpath(input_path.name) + ) + + @pytest.mark.parametrize( "model_path,subview_sep,variables,modules,n_deps,dep_vars", [ @@ -33,7 +42,7 @@ [".", "-"], ["variablex"], ["view_3/subview_1", "view_1/submodule_1"], - (12, 0, 1, 1, 1), + (12, 0, 1, 1, 2), {"another_var": 5, "look_up_definition": 3} ) ], @@ -56,7 +65,7 @@ class TestSubmodel: } @pytest.fixture - def model(self, shared_tmpdir, model_path, subview_sep, _root): + def model(self, shared_tmpdir, model_path, subview_sep, _root, input_file): """ Translate the model or read a translated version. This way each file is only translated once. @@ -117,13 +126,17 @@ def test_select_submodel(self, model, variables, modules, # assert original stateful elements assert len(model._dynamicstateful_elements) == 2 assert "_integ_other_stock" in model._stateful_elements - assert "_integ_other_stock" in model.components._dependencies - assert "other_stock" in model.components._dependencies - assert "other stock" in model.components._namespace + assert "_integ_other_stock" in model._dependencies + assert "other_stock" in model._dependencies + assert "other stock" in model._namespace + assert "other stock" in model._doc["Real Name"].to_list() + assert "other_stock" in model._doc["Py Name"].to_list() assert "_integ_stock" in model._stateful_elements - assert "_integ_stock" in model.components._dependencies - assert "stock" in model.components._dependencies - assert "Stock" in model.components._namespace + assert "_integ_stock" in model._dependencies + assert "stock" in model._dependencies + assert "Stock" in model._namespace + assert "Stock" in model._doc["Real Name"].to_list() + assert "stock" in model._doc["Py Name"].to_list() # select submodel with pytest.warns(UserWarning) as record: @@ -135,29 +148,31 @@ def test_select_submodel(self, model, variables, modules, # assert stateful elements change assert len(model._dynamicstateful_elements) == 1 assert "_integ_other_stock" not in model._stateful_elements - assert "_integ_other_stock" not in model.components._dependencies - assert "other_stock" not in model.components._dependencies - assert "other stock" not in model.components._namespace + assert "_integ_other_stock" not in model._dependencies + assert "other_stock" not in model._dependencies + assert "other stock" not in model._namespace + assert "other stock" not in model._doc["Real Name"].to_list() + assert "other_stock" not in model._doc["Py Name"].to_list() assert "_integ_stock" in model._stateful_elements - assert "_integ_stock" in model.components._dependencies - assert "stock" in model.components._dependencies - assert "Stock" in model.components._namespace + assert "_integ_stock" in model._dependencies + assert "stock" in model._dependencies + assert "Stock" in model._namespace + assert "Stock" in model._doc["Real Name"].to_list() + assert "stock" in model._doc["Py Name"].to_list() if not dep_vars: # totally independent submodels can run without producing # nan values - assert len(record) == 1 assert not np.any(np.isnan(model.run())) else: # running the model without redefining dependencies will # produce nan values - assert len(record) == 2 assert "Exogenous components for the following variables are"\ - + " necessary but not given:" in str(record[1].message) + + " necessary but not given:" in str(record[-1].message) assert "Please, set them before running the model using "\ - + "set_components method..." in str(record[1].message) + + "set_components method..." in str(record[-1].message) for var in dep_vars: - assert var in str(record[1].message) + assert var in str(record[-1].message) assert np.any(np.isnan(model.run())) # redefine dependencies assert not np.any(np.isnan(model.run(params=dep_vars))) @@ -168,7 +183,6 @@ def test_select_submodel(self, model, variables, modules, model.select_submodel(vars=variables, modules=modules, exogenous_components=dep_vars) - assert len(record) == 1 assert not np.any(np.isnan(model.run())) @@ -196,7 +210,7 @@ def test_select_submodel(self, model, variables, modules, ) class TestGetVarsInModuleErrors: @pytest.fixture - def model(self, shared_tmpdir, model_path, split_views, _root): + def model(self, shared_tmpdir, model_path, split_views, _root, input_file): """ Translate the model. """ diff --git a/tests/pytest_translation/vensim2py/pytest_split_views.py b/tests/pytest_translators/pytest_split_views.py similarity index 87% rename from tests/pytest_translation/vensim2py/pytest_split_views.py rename to tests/pytest_translators/pytest_split_views.py index 1267bfe1..4852f3ed 100644 --- a/tests/pytest_translation/vensim2py/pytest_split_views.py +++ b/tests/pytest_translators/pytest_split_views.py @@ -7,6 +7,15 @@ from pysd.tools.benchmarking import assert_frames_close +@pytest.fixture(scope="class") +def input_file(shared_tmpdir, _root): + input_path = Path("more-tests/split_model/input.xlsx") + shutil.copy( + _root.joinpath(input_path), + shared_tmpdir.joinpath(input_path.name) + ) + + @pytest.mark.parametrize( "model_path,subview_sep,modules,macros,original_vars,py_vars," + "stateful_objs", @@ -72,7 +81,7 @@ class TestSplitViews: Test for splitting Vensim views in modules and submodules """ @pytest.fixture - def model_file(self, shared_tmpdir, model_path): + def model_file(self, shared_tmpdir, model_path, input_file): return shared_tmpdir.joinpath(model_path.name) @pytest.fixture @@ -85,9 +94,7 @@ def expected_files(self, shared_tmpdir, _root, model_path, ) modules_dir = shared_tmpdir.joinpath("modules_" + model_name) files = { - shared_tmpdir.joinpath("_namespace_" + model_name + ".json"), shared_tmpdir.joinpath("_subscripts_" + model_name + ".json"), - shared_tmpdir.joinpath("_dependencies_" + model_name + ".json"), modules_dir.joinpath("_modules.json") } [files.add(modules_dir.joinpath(module + ".py")) for module in modules] @@ -112,10 +119,10 @@ def test_read_vensim_split_model(self, model_file, subview_sep, assert file.is_file(), f"File {file} has not been created..." # check the dictionaries - assert isinstance(model_split.components._namespace, dict) - assert isinstance(model_split.components._subscript_dict, dict) - assert isinstance(model_split.components._dependencies, dict) - assert isinstance(model_split.components._modules, dict) + assert isinstance(model_split._namespace, dict) + assert isinstance(model_split._subscript_dict, dict) + assert isinstance(model_split._dependencies, dict) + assert isinstance(model_split._modules, dict) # assert taht main modules are dictionary keys for module in modules: @@ -124,7 +131,7 @@ def test_read_vensim_split_model(self, model_file, subview_sep, # assert that original variables are in the namespace for var in original_vars: - assert var in model_split.components._namespace.keys() + assert var in model_split._namespace.keys() # assert that the functions are not defined in the main file model_py_file = model_file.with_suffix(".py") @@ -165,8 +172,14 @@ def test_read_vensim_split_model(self, model_file, subview_sep, ["a"], "The given subview separators were not matched in any view name." ), + ( # no_sketch + Path("more-tests/not_implemented_and_incomplete/" + "test_not_implemented_and_incomplete.mdl"), + ["a"], + "No sketch detected. The model will be built in a single file." + ), ], - ids=["warning_noviews", "not_match_separator"] + ids=["warning_noviews", "not_match_separator", "no_sketch"] ) class TestSplitViewsWarnings: """ diff --git a/tests/pytest_translators/pytest_vensim.py b/tests/pytest_translators/pytest_vensim.py new file mode 100644 index 00000000..5d438a85 --- /dev/null +++ b/tests/pytest_translators/pytest_vensim.py @@ -0,0 +1,148 @@ +import pytest +from pathlib import Path +from parsimonious import VisitationError + +from pysd.translators.vensim.vensim_file import VensimFile +from pysd.translators.vensim.vensim_element import Element + + +@pytest.mark.parametrize( + "path", + [ + ( # teacup + "samples/teacup/teacup.mdl" + ), + ( # macros + "tests/macro_multi_expression/test_macro_multi_expression.mdl" + ), + ( # mapping + "tests/subscript_mapping_vensim/test_subscript_mapping_vensim.mdl" + ), + ( # data + "tests/data_from_other_model/test_data_from_other_model.mdl" + ), + ( # except + "tests/except/test_except.mdl" + ) + ], + ids=["teacup", "macros", "mapping", "data", "except"] +) +class TestVensimFile: + """ + Test for splitting Vensim views in modules and submodules + """ + @pytest.fixture + def model_path(self, _root, path): + return _root.joinpath("test-models").joinpath(path) + + @pytest.mark.dependency(name="read_vensim_file") + def test_read_vensim_file(self, model_path): + # assert that the files don't exist in the temporary directory + ven_file = VensimFile(model_path) + + assert hasattr(ven_file, "mdl_path") + assert hasattr(ven_file, "root_path") + assert hasattr(ven_file, "model_text") + + assert isinstance(getattr(ven_file, "mdl_path"), Path) + assert isinstance(getattr(ven_file, "root_path"), Path) + assert isinstance(getattr(ven_file, "model_text"), str) + + @pytest.mark.dependency(depends=["read_vensim_file"]) + def test_file_split_file_sections(self, model_path): + ven_file = VensimFile(model_path) + ven_file.parse() + + +class TestElements: + """ + Test for splitting Vensim views in modules and submodules + """ + @pytest.fixture + def element(self, equation): + return Element(equation, "", "") + + @pytest.mark.parametrize( + "equation,error_message", + [ + ( # no-letter + "dim: (1-12)", + "A numeric range must contain at least one letter." + ), + ( # greater + "dim: (a12-a10)", + "The number of the first subscript value must be lower " + "than the second subscript value in a subscript numeric" + " range." + ), + ( # different-leading + "dim: (aba1-abc12)", + "Only matching names ending in numbers are valid." + ), + ], + ids=["no-letter", "greater", "different-leading"] + ) + def test_subscript_range_error(self, element, error_message): + # assert that the files don't exist in the temporary directory + with pytest.raises(VisitationError, match=error_message): + element.parse() + + @pytest.mark.parametrize( + "equation,mapping", + [ + ( # single + "subcon : subcon1,subcon2->(del: del con1, del con2)", + ["del"] + ), + ( # single2 + "subcon : subcon1,subcon2 -> (del:del con1,del con2)", + ["del"] + ), + ( # multiple + "class: class1,class2->(metal:class1 metal,class2 metal)," + "(our metal:ourC1,ourC2)", + ["metal", "our metal"] + ), + ( # multiple2 + "class: class1,class2-> (metal:class1 metal,class2 metal) ," + " (our metal:ourC1,ourC2)", + ["metal", "our metal"] + ), + ], + ids=["single", "single2", "multiple", "multiple2"] + ) + def test_complex_mapping(self, element, mapping): + # parse the mapping + warning_message = r"Subscript mapping detected\. "\ + r"This feature works only for simple cases\." + with pytest.warns(UserWarning, match=warning_message): + out = element.parse() + + assert out.mapping == mapping + + @pytest.mark.parametrize( + "equation,mapping", + [ + ( # single + "subcon : subcon1,subcon2 -> del", + ["del"] + ), + ( # single2 + "subcon : subcon1,subcon2->del", + ["del"] + ), + ( # multiple + "class: class1,class2->metal,our metal", + ["metal", "our metal"] + ), + ( # multiple2 + "class: class1,class2->metal , our metal", + ["metal", "our metal"] + ), + ], + ids=["single", "single2", "multiple", "multiple2"] + ) + def test_simple_mapping(self, element, mapping): + # parse the mapping + out = element.parse() + assert out.mapping == mapping diff --git a/tests/pytest_types/data/pytest_data.py b/tests/pytest_types/data/pytest_data.py index 4aebd8d1..ab052702 100644 --- a/tests/pytest_types/data/pytest_data.py +++ b/tests/pytest_types/data/pytest_data.py @@ -1,42 +1,11 @@ import pytest import xarray as xr +import pandas as pd -from pysd.py_backend.data import Data +from pysd.py_backend.data import Data, TabData -@pytest.mark.parametrize( - "value,interp,raise_type,error_message", - [ - ( # not_loaded_data - None, - "interpolate", - ValueError, - "Trying to interpolate data variable before loading the data..." - ), - # test that try/except block on call doesn't catch errors differents - # than data = None - ( # try_except_1 - 3, - None, - TypeError, - "'int' object is not subscriptable" - ), - ( # try_except_2 - xr.DataArray([10, 20], {'dim1': [0, 1]}, ['dim1']), - None, - KeyError, - "'time'" - ), - ( # try_except_3 - xr.DataArray([10, 20], {'time': [0, 1]}, ['time']), - None, - AttributeError, - "'Data' object has no attribute 'is_float'" - ) - ], - ids=["not_loaded_data", "try_except_1", "try_except_2", "try_except_3"] -) @pytest.mark.filterwarnings("ignore") class TestDataErrors(): # Test errors associated with Data class @@ -51,6 +20,123 @@ def data(self, value, interp): obj.py_name = "data" return obj + @pytest.mark.parametrize( + "value,interp,raise_type,error_message", + [ + ( # not_loaded_data + None, + "interpolate", + ValueError, + "Trying to interpolate data variable before loading " + "the data..." + ), + # test that try/except block on call doesn't catch errors + # differents than data = None + ( # try_except + xr.DataArray([10, 20], {'time': [0, 1]}, ['time']), + None, + AttributeError, + "'Data' object has no attribute 'is_float'" + ) + ], + ids=["not_loaded_data", "try_except"] + ) def test_data_errors(self, data, raise_type, error_message): with pytest.raises(raise_type, match=error_message): data(1.5) + + def test_invalid_interp_method(self): + error_message = r"\nThe interpolation method \(interp\) must be"\ + r" 'raw', 'interpolate', 'look_forward' or 'hold_backward'" + with pytest.raises(ValueError, match=error_message): + TabData("", "", {}, interp="invalid") + + +@pytest.mark.parametrize( + "value,new_value,expected", + [ + ( # float-constant + xr.DataArray([10, 20], {'time': [0, 1]}, ['time']), + 26, + 26 + ), + ( # float-series + xr.DataArray([10, 20], {'time': [0, 1]}, ['time']), + pd.Series(index=[1, 20, 40], data=[2, 10, 2]), + xr.DataArray([2, 10, 2], {"time": [1, 20, 40]}, ["time"]) + ), + ( # array-constantfloat + xr.DataArray( + [[10, 20], [30, 40]], + {"time": [0, 1], "dim":["A", "B"]}, + ["time", "dim"]), + 26, + xr.DataArray(26, {"dim": ["A", "B"]}, ["dim"]), + ), + ( # array-seriesfloat + xr.DataArray( + [[10, 20], [30, 40]], + {"time": [0, 1], "dim":["A", "B"]}, + ["time", "dim"]), + pd.Series(index=[1, 20, 40], data=[2, 10, 2]), + xr.DataArray( + [[2, 2], [10, 10], [2, 2]], + {"time": [1, 20, 40], "dim":["A", "B"]}, + ["time", "dim"]) + ), + ( # array-constantarray + xr.DataArray( + [[[10, 20], [30, 40]], [[15, 25], [35, 45]]], + {"time": [0, 1], "dim":["A", "B"], "dim2": ["C", "D"]}, + ["time", "dim", "dim2"]), + xr.DataArray( + [1, 2], + {"dim": ["A", "B"]}, + ["dim"]), + xr.DataArray( + [[1, 2], [1, 2]], + {"dim": ["A", "B"], "dim2": ["C", "D"]}, + ["dim", "dim2"]) + ), + ( # array-seriesarray + xr.DataArray( + [[[10, 20], [30, 40]], [[15, 25], [35, 45]]], + {"time": [0, 1], "dim":["A", "B"], "dim2": ["C", "D"]}, + ["time", "dim", "dim2"]), + pd.Series(index=[1, 20, 40], data=[ + xr.DataArray([1, 2], {"dim": ["A", "B"]}, ["dim"]), + xr.DataArray([10, 20], {"dim": ["A", "B"]}, ["dim"]), + xr.DataArray([1, 2], {"dim": ["A", "B"]}, ["dim"]) + ]), + xr.DataArray( + [[[1, 2], [1, 2]], [[10, 20], [10, 20]], [[1, 2], [1, 2]]], + {"time": [1, 20, 40], "dim":["A", "B"], "dim2": ["C", "D"]}, + ["time", "dim", "dim2"]) + ) + ], + ids=[ + "float-constant", "float-series", + "array-constantfloat", "array-seriesfloat", + "array-constantarray", "array-seriesarray" + ] +) +class TestDataSetValues(): + + @pytest.fixture + def data(self, value): + obj = Data() + obj.data = value + obj.interp = "interp" + obj.is_float = len(value.shape) < 2 + obj.final_coords = { + dim: value.coords[dim] for dim in value.dims if dim != "time" + } + obj.py_name = "data" + return obj + + def test_data_set_value(self, data, new_value, expected): + data.set_values(new_value) + if isinstance(expected, (float, int)): + assert data.data == expected + else: + assert data.data.equals(expected) diff --git a/tests/pytest_types/data/pytest_data_with_model.py b/tests/pytest_types/data/pytest_data_with_model.py index acb3f5b9..de87ee3f 100644 --- a/tests/pytest_types/data/pytest_data_with_model.py +++ b/tests/pytest_types/data/pytest_data_with_model.py @@ -94,6 +94,17 @@ def test_get_data_and_run(self, model, expected): model.run(return_columns=["var1", "var2", "var3"]), expected) + def test_modify_data(self, model, expected): + out = model.run(params={ + "var1": pd.Series(index=[1, 3, 7], data=[10, 20, 30]), + "var2": 10 + }) + + assert (out["var2"] == 10).all() + assert ( + out["var1"] == [10, 10, 15, 20, 22.5, 25, 27.5, 30, 30, 30, 30] + ).all() + class TestPySDDataErrors: def model(self, data_model, data_files, shared_tmpdir): diff --git a/tests/pytest_types/lookup/pytest_lookups.py b/tests/pytest_types/lookup/pytest_lookups.py new file mode 100644 index 00000000..490fc025 --- /dev/null +++ b/tests/pytest_types/lookup/pytest_lookups.py @@ -0,0 +1,105 @@ +import pytest + +import xarray as xr +import pandas as pd + +from pysd.py_backend.lookups import Lookups + + +@pytest.mark.parametrize( + "value,new_value,expected", + [ + ( # float-constant + xr.DataArray([10, 20], {'lookup_dim': [0, 1]}, ['lookup_dim']), + 26, + 26 + ), + ( # float-series + xr.DataArray([10, 20], {'lookup_dim': [0, 1]}, ['lookup_dim']), + pd.Series(index=[1, 20, 40], data=[2, 10, 2]), + xr.DataArray( + [2, 10, 2], + {"lookup_dim": [1, 20, 40]}, + ["lookup_dim"] + ) + + ), + ( # array-constantfloat + xr.DataArray( + [[10, 20], [30, 40]], + {"lookup_dim": [0, 1], "dim":["A", "B"]}, + ["lookup_dim", "dim"]), + 26, + xr.DataArray(26, {"dim": ["A", "B"]}, ["dim"]), + ), + ( # array-seriesfloat + xr.DataArray( + [[10, 20], [30, 40]], + {"lookup_dim": [0, 1], "dim":["A", "B"]}, + ["lookup_dim", "dim"]), + pd.Series(index=[1, 20, 40], data=[2, 10, 2]), + xr.DataArray( + [[2, 2], [10, 10], [2, 2]], + {"lookup_dim": [1, 20, 40], "dim":["A", "B"]}, + ["lookup_dim", "dim"]) + ), + ( # array-constantarray + xr.DataArray( + [[[10, 20], [30, 40]], [[15, 25], [35, 45]]], + {"lookup_dim": [0, 1], "dim":["A", "B"], "dim2": ["C", "D"]}, + ["lookup_dim", "dim", "dim2"]), + xr.DataArray( + [1, 2], + {"dim": ["A", "B"]}, + ["dim"]), + xr.DataArray( + [[1, 2], [1, 2]], + {"dim": ["A", "B"], "dim2": ["C", "D"]}, + ["dim", "dim2"]) + ), + ( # array-seriesarray + xr.DataArray( + [[[10, 20], [30, 40]], [[15, 25], [35, 45]]], + {"lookup_dim": [0, 1], "dim":["A", "B"], "dim2": ["C", "D"]}, + ["lookup_dim", "dim", "dim2"]), + pd.Series(index=[1, 20, 40], data=[ + xr.DataArray([1, 2], {"dim": ["A", "B"]}, ["dim"]), + xr.DataArray([10, 20], {"dim": ["A", "B"]}, ["dim"]), + xr.DataArray([1, 2], {"dim": ["A", "B"]}, ["dim"]) + ]), + xr.DataArray( + [[[1, 2], [1, 2]], [[10, 20], [10, 20]], [[1, 2], [1, 2]]], + { + "lookup_dim": [1, 20, 40], + "dim":["A", "B"], + "dim2": ["C", "D"] + }, + ["lookup_dim", "dim", "dim2"]) + ) + ], + ids=[ + "float-constant", "float-series", + "array-constantfloat", "array-seriesfloat", + "array-constantarray", "array-seriesarray" + ] +) +class TestLookupsSetValues(): + + @pytest.fixture + def lookups(self, value): + obj = Lookups() + obj.data = value + obj.interp = "interp" + obj.is_float = len(value.shape) < 2 + obj.final_coords = { + dim: value.coords[dim] for dim in value.dims if dim != "lookup_dim" + } + obj.py_name = "lookup" + return obj + + def test_lookups_set_value(self, lookups, new_value, expected): + lookups.set_values(new_value) + if isinstance(expected, (float, int)): + assert lookups.data == expected + else: + assert lookups.data.equals(expected) diff --git a/tests/test-models b/tests/test-models index de294a8a..1502dce4 160000 --- a/tests/test-models +++ b/tests/test-models @@ -1 +1 @@ -Subproject commit de294a8ad0f2c1a2bf41c351cfc4ab637bc39825 +Subproject commit 1502dce4b5dbe8d86e6f310fc40f5c33a6dea1ec diff --git a/tests/unit_test_benchmarking.py b/tests/unit_test_benchmarking.py index 696c6c46..acea887a 100644 --- a/tests/unit_test_benchmarking.py +++ b/tests/unit_test_benchmarking.py @@ -1,10 +1,11 @@ -import os +from pathlib import Path from unittest import TestCase # most of the features of this script are already tested indirectly when # running vensim and xmile integration tests -_root = os.path.dirname(__file__) +_root = Path(__file__).parent +test_model = _root.joinpath("test-models/samples/teacup/teacup.mdl") class TestErrors(TestCase): @@ -13,7 +14,7 @@ def test_canonical_file_not_found(self): from pysd.tools.benchmarking import runner with self.assertRaises(FileNotFoundError) as err: - runner(os.path.join(_root, "more-tests/not_existent.mdl")) + runner(_root.joinpath("more-tests/not_existent.mdl")) self.assertIn( 'Canonical output file not found.', @@ -23,12 +24,11 @@ def test_non_valid_model(self): from pysd.tools.benchmarking import runner with self.assertRaises(ValueError) as err: - runner(os.path.join( - _root, - "more-tests/not_vensim/test_not_vensim.txt")) + runner(_root.joinpath("more-tests/not_vensim/test_not_vensim.txt")) self.assertIn( - 'Modelfile should be *.mdl or *.xmile', + "The model file name must be a Vensim (.mdl), a Xmile " + "(.xmile, .xml, .stmx) or a PySD (.py) model file...", str(err.exception)) def test_different_frames_error(self): @@ -36,9 +36,8 @@ def test_different_frames_error(self): with self.assertRaises(AssertionError) as err: assert_frames_close( - load_outputs(os.path.join(_root, "data/out_teacup.csv")), - load_outputs( - os.path.join(_root, "data/out_teacup_modified.csv"))) + load_outputs(_root.joinpath("data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup_modified.csv"))) self.assertIn( "Following columns are not close:\n\tTeacup Temperature", @@ -58,9 +57,8 @@ def test_different_frames_error(self): with self.assertRaises(AssertionError) as err: assert_frames_close( - load_outputs(os.path.join(_root, "data/out_teacup.csv")), - load_outputs( - os.path.join(_root, "data/out_teacup_modified.csv")), + load_outputs(_root.joinpath("data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup_modified.csv")), verbose=True) self.assertIn( @@ -85,9 +83,8 @@ def test_different_frames_warning(self): with catch_warnings(record=True) as ws: assert_frames_close( - load_outputs(os.path.join(_root, "data/out_teacup.csv")), - load_outputs( - os.path.join(_root, "data/out_teacup_modified.csv")), + load_outputs(_root.joinpath("data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup_modified.csv")), assertion="warn") # use only user warnings @@ -112,9 +109,8 @@ def test_different_frames_warning(self): with catch_warnings(record=True) as ws: assert_frames_close( - load_outputs(os.path.join(_root, "data/out_teacup.csv")), - load_outputs( - os.path.join(_root, "data/out_teacup_modified.csv")), + load_outputs(_root.joinpath("data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup_modified.csv")), assertion="warn", verbose=True) # use only user warnings @@ -137,6 +133,18 @@ def test_different_frames_warning(self): "Expected values:\n\t", str(wu[0].message)) + def test_different_frames_return(self): + from pysd.tools.benchmarking import load_outputs, assert_frames_close + + cols, first_false_time, first_false_cols = assert_frames_close( + load_outputs(_root.joinpath("data/out_teacup.csv")), + load_outputs(_root.joinpath("data/out_teacup_modified.csv")), + assertion="return") + + assert cols == {"Teacup Temperature"} + assert first_false_time == 30. + assert first_false_cols == {"Teacup Temperature"} + def test_different_cols(self): from warnings import catch_warnings from pysd.tools.benchmarking import assert_frames_close @@ -218,3 +226,10 @@ def test_invalid_input(self): self.assertIn( "Inputs must both be pandas DataFrames.", str(err.exception)) + + def test_run_python(self): + from pysd.tools.benchmarking import runner + assert ( + runner(str(test_model))[0] + == runner(test_model.with_suffix(".py"))[0] + ).all().all() diff --git a/tests/unit_test_builder.py b/tests/unit_test_builder.py deleted file mode 100644 index 313f0b9b..00000000 --- a/tests/unit_test_builder.py +++ /dev/null @@ -1,430 +0,0 @@ -import textwrap - -from unittest import TestCase - -import numpy as np -from numbers import Number -import xarray as xr - - -def runner(string, ns=None): - code = compile(string, '', 'exec') - if not ns: - ns = dict() - ns.update({'xr': xr, 'np': np}) - exec(code, ns) - return ns - - -class TestBuildElement(TestCase): - def test_no_subs_constant(self): - from pysd.translation.builder import build_element - string = textwrap.dedent( - build_element(element={'kind': 'constant', - 'subs': [[]], - 'merge_subs': [], - 'doc': '', - 'py_name': 'my_variable', - 'real_name': 'My Variable', - 'py_expr': ['0.01'], - 'unit': '', - 'eqn': '', - 'lims': '', - 'arguments': ''}, - subscript_dict={}) - ) - ns = runner(string) - a = ns['my_variable']() - self.assertIsInstance(a, Number) - self.assertEqual(a, .01) - - def test_no_subs_call(self): - from pysd.translation.builder import build_element - string = textwrap.dedent( - build_element(element={'kind': 'constant', - 'subs': [[]], - 'merge_subs': [], - 'doc': '', - 'py_name': 'my_first_variable', - 'real_name': 'My Variable', - 'py_expr': ['other_variable()'], - 'eqn': '', - 'lims': '', - 'unit': '', - 'arguments': ''}, - subscript_dict={}) - ) - ns = {'other_variable': lambda: 3} - ns = runner(string, ns) - a = ns['my_first_variable']() - self.assertIsInstance(a, Number) - self.assertEqual(a, 3) - - -class TestBuildFunctionCall(TestCase): - def test_build_function_not_implemented(self): - from warnings import catch_warnings - from pysd.translation.builder import build_function_call - args = ['a', 'b'] - nif = {"name": "not_implemented_function", - "module": "functions", - "original_name": "NIF"} - with catch_warnings(record=True) as ws: - self.assertEqual(build_function_call(nif, args), - "not_implemented_function('NIF',a,b)") - self.assertEqual(len(ws), 1) - self.assertTrue("Trying to translate NIF which it is " - + "not implemented on PySD." - in str(ws[0].message)) - - def test_build_function_with_time_dependency(self): - from pysd.translation.builder import build_function_call - args = ['a', 'b'] - pulse = { - "name": "pulse", - "parameters": [ - {"name": "time", "type": "time"}, - {"name": "start"}, - {"name": "duration"}, - ], - "module": "functions", - } - - dependencies = {'a': 1, 'b': 2} - self.assertNotIn('time', dependencies) - self.assertEqual(build_function_call(pulse, args, dependencies), - "pulse(__data['time'], a, b)") - self.assertIn('time', dependencies) - - def test_build_function_ignore_arguments(self): - from pysd.translation.builder import build_function_call - args = ['a', 'b', 'c'] - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A"}, - {"name": "B", "type": "ignore"}, - {"name": "C"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, args), - "my_func(a, c)") - - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A", "type": "ignore"}, - {"name": "B"}, - {"name": "C", "type": "ignore"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, args), - "my_func(b)") - - def test_build_function_lambda_arguments(self): - from pysd.translation.builder import build_function_call - args = ['a', 'b', 'c'] - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A"}, - {"name": "B", "type": "lambda"}, - {"name": "C"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, args), - "my_func(a, lambda: b, c)") - - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A", "type": "lambda"}, - {"name": "B"}, - {"name": "C", "type": "lambda"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, args), - "my_func(lambda: a, b, lambda: c)") - - def test_build_function_optional_arguments(self): - from pysd.translation.builder import build_function_call - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A"}, - {"name": "B"}, - {"name": "C", "optional": True} - ] - } - - self.assertEqual(build_function_call(my_func_conf, ['a', 'b', 'c']), - "my_func(a, b, c)") - - self.assertEqual(build_function_call(my_func_conf, ['a', 'b']), - "my_func(a, b)") - - # optional lambda argument, check optional + type - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A"}, - {"name": "B"}, - {"name": "C", "type": "lambda", "optional": True} - ] - } - - self.assertEqual(build_function_call(my_func_conf, ['a', 'b', 'c']), - "my_func(a, b, lambda: c)") - - self.assertEqual(build_function_call(my_func_conf, ['a', 'b']), - "my_func(a, b)") - - def test_build_function_predef_arguments(self): - from pysd.translation.builder import build_function_call - args = ['a', 'c'] - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "A"}, - {"name": "0", "type": "predef"}, - {"name": "C"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, args), - "my_func(a, 0, c)") - - my_func_conf = { - "name": "my_func", - "parameters": [ - {"name": "time_step()", "type": "predef"}, - {"name": "B"}, - {"name": "1", "type": "predef"} - ] - } - - self.assertEqual(build_function_call(my_func_conf, ["b"]), - "my_func(time_step(), b, 1)") - - -class TestBuild(TestCase): - def test_build(self): - # Todo: add other builder-specific inclusions to this test - from pysd.translation.builder import build - actual = textwrap.dedent( - build(elements=[{'kind': 'component', - 'subs': [], - 'doc': '', - 'py_name': 'stocka', - 'real_name': 'StockA', - 'py_expr': ["_state['stocka']"], - 'eqn': [''], - 'lims': '', - 'unit': '', - 'merge_subs': [], - 'arguments': ''}, - {'kind': 'component', - 'subs': [], - 'doc': 'Provides derivative for stocka', - 'py_name': '_dstocka_dt', - 'real_name': 'Implicit', - 'py_expr': ['flowa()'], - 'unit': 'See docs for stocka', - 'eqn': [''], - 'lims': '', - 'merge_subs': [], - 'arguments': ''}, - {'kind': 'setup', - 'subs': [], - 'doc': 'Provides initial conditions for stocka', - 'py_name': 'init_stocka', - 'real_name': 'Implicit', - 'py_expr': ['-10'], - 'unit': 'See docs for stocka', - 'eqn': [''], - 'lims': '', - 'merge_subs': [], - 'arguments': ''}], - namespace={'StockA': 'stocka'}, - subscript_dict={'Dim1': ['A', 'B', 'C']}, - dependencies={ - "stocka": {"_integ_stocka"}, - "_integ_stocka": { - "initial": None, - "step": {"flowa"} - }, - "flowa": None - }, - outfile_name='return')) - self.assertIn('_subscript_dict = {"Dim1": ["A", "B", "C"]}', actual) - self.assertIn('_namespace = {"StockA": "stocka"}', actual) - self.assertIn( - '_dependencies = {\n "stocka": {"_integ_stocka"},' - + '\n "_integ_stocka": {"initial": None, "step": {"flowa"}},' - + '\n "flowa": None,\n}', actual) - - -class TestMergePartialElements(TestCase): - def test_single_set(self): - from pysd.translation.builder import merge_partial_elements - - self.assertEqual( - merge_partial_elements( - [{'py_name': 'a', 'py_expr': 'ms', - 'subs': ['Name1', 'element1'], - 'merge_subs': ['Name1', 'Elements'], - 'real_name': 'A', 'doc': 'Test', 'unit': None, - 'eqn': 'eq1', 'lims': '', - 'dependencies': {'b': 1, 'time': 3}, - 'kind': 'component', 'arguments': ''}, - {'py_name': 'a', 'py_expr': 'njk', - 'subs': ['Name1', 'element2'], - 'merge_subs': ['Name1', 'Elements'], - 'real_name': 'A', 'doc': None, 'unit': None, - 'eqn': 'eq2', 'lims': '', - 'dependencies': {'c': 1, 'time': 5}, - 'kind': 'component', 'arguments': ''}, - {'py_name': 'a', 'py_expr': 'as', - 'subs': ['Name1', 'element3'], - 'merge_subs': ['Name1', 'Elements'], - 'real_name': 'A', 'doc': '', 'unit': None, - 'eqn': 'eq3', 'lims': '', 'dependencies': {'b': 1}, - 'kind': 'component', 'arguments': ''}]), - [{'py_name': 'a', - 'py_expr': ['ms', 'njk', 'as'], - 'subs': [['Name1', 'element1'], - ['Name1', 'element2'], - ['Name1', 'element3']], - 'merge_subs': ['Name1', 'Elements'], - 'kind': 'component', - 'doc': 'Test', - 'real_name': 'A', - 'unit': None, - 'eqn': ['eq1', 'eq2', 'eq3'], - 'lims': '', - 'parent_name': None, - 'dependencies': {'b': 2, 'c': 1, 'time': 8}, - 'arguments': '' - }]) - - def test_multiple_sets(self): - from pysd.translation.builder import merge_partial_elements - actual = merge_partial_elements( - [{'py_name': 'a', 'py_expr': 'ms', 'subs': ['Name1', 'element1'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': {'b': 1}, - 'real_name': 'A', 'doc': 'Test', 'unit': None, - 'eqn': 'eq1', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': 'a', 'py_expr': 'njk', 'subs': ['Name1', 'element2'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': {'b': 2}, - 'real_name': 'A', 'doc': None, 'unit': None, - 'eqn': 'eq2', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': 'a', 'py_expr': 'as', 'subs': ['Name1', 'element3'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': {'b': 1}, - 'real_name': 'A', 'doc': '', 'unit': None, - 'eqn': 'eq3', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': '_b', 'py_expr': 'bgf', 'subs': ['Name1', 'element1'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': { - 'initial': {'c': 3}, 'step': {}}, - 'real_name': 'B', 'doc': 'Test', 'unit': None, - 'eqn': 'eq4', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': '_b', 'py_expr': 'r4', 'subs': ['Name1', 'element2'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': { - 'initial': {'d': 1}, 'step': {'time': 2, 'd': 5}}, - 'real_name': 'B', 'doc': None, 'unit': None, - 'eqn': 'eq5', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': '_b', 'py_expr': 'ymt', 'subs': ['Name1', 'element3'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': { - 'initial': {}, 'step': {'time': 3, 'a': 1}}, - 'real_name': 'B', 'doc': '', 'unit': None, - 'eqn': 'eq6', 'lims': '', 'kind': 'component', 'arguments': ''}]) - - expected = [{'py_name': 'a', - 'py_expr': ['ms', 'njk', 'as'], - 'subs': [['Name1', 'element1'], - ['Name1', 'element2'], - ['Name1', 'element3']], - 'merge_subs': ['Name1', 'Elements'], - 'kind': 'component', - 'doc': 'Test', - 'real_name': 'A', - 'unit': None, - 'eqn': ['eq1', 'eq2', 'eq3'], - 'lims': '', - 'parent_name': None, - 'dependencies': {'b': 4}, - 'arguments': '' - }, - {'py_name': '_b', - 'py_expr': ['bgf', 'r4', 'ymt'], - 'subs': [['Name1', 'element1'], - ['Name1', 'element2'], - ['Name1', 'element3']], - 'merge_subs': ['Name1', 'Elements'], - 'kind': 'component', - 'doc': 'Test', - 'real_name': 'B', - 'unit': None, - 'eqn': ['eq4', 'eq5', 'eq6'], - 'lims': '', - 'parent_name': None, - 'dependencies': { - 'initial': {'c': 3, 'd': 1}, - 'step': {'time': 5, 'a': 1, 'd': 5} - }, - 'arguments': '' - }] - self.assertIn(actual[0], expected) - self.assertIn(actual[1], expected) - - def test_non_set(self): - from pysd.translation.builder import merge_partial_elements - actual = merge_partial_elements( - [{'py_name': 'a', 'py_expr': 'ms', 'subs': ['Name1', 'element1'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': {'c': 1}, - 'real_name': 'A', 'doc': 'Test', 'unit': None, - 'eqn': 'eq1', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': 'a', 'py_expr': 'njk', 'subs': ['Name1', 'element2'], - 'merge_subs': ['Name1', 'Elements'], 'dependencies': {'b': 2}, - 'real_name': 'A', 'doc': None, 'unit': None, - 'eqn': 'eq2', 'lims': '', 'kind': 'component', 'arguments': ''}, - {'py_name': 'c', 'py_expr': 'as', 'subs': ['Name1', 'element3'], - 'merge_subs': ['Name1', 'elements3'], 'dependencies': {}, - 'real_name': 'C', 'doc': 'hi', 'unit': None, - 'eqn': 'eq3', 'lims': '', 'kind': 'component', 'arguments': ''}, - ]) - - expected = [{'py_name': 'a', - 'py_expr': ['ms', 'njk'], - 'subs': [['Name1', 'element1'], ['Name1', 'element2']], - 'merge_subs': ['Name1', 'Elements'], - 'kind': 'component', - 'doc': 'Test', - 'real_name': 'A', - 'unit': None, - 'eqn': ['eq1', 'eq2'], - 'lims': '', - 'parent_name': None, - 'dependencies': {'b': 2, 'c': 1}, - 'arguments': '' - }, - {'py_name': 'c', - 'py_expr': ['as'], - 'subs': [['Name1', 'element3']], - 'merge_subs': ['Name1', 'elements3'], - 'kind': 'component', - 'doc': 'hi', - 'real_name': 'C', - 'unit': None, - 'eqn': ['eq3'], - 'lims': '', - 'parent_name': None, - 'dependencies': {}, - 'arguments': '' - }] - - self.assertIn(actual[0], expected) - self.assertIn(actual[1], expected) diff --git a/tests/unit_test_cli.py b/tests/unit_test_cli.py index 36754f9e..b7ae522a 100644 --- a/tests/unit_test_cli.py +++ b/tests/unit_test_cli.py @@ -67,8 +67,8 @@ def test_read_not_model(self): self.assertNotEqual(out.returncode, 0) self.assertIn(f"PySD: error: when parsing {model}", stderr) self.assertIn( - "The model file name must be Vensim (.mdl), Xmile (.xmile) " - "or PySD (.py) model file...", stderr) + "The model file name must be a Vensim (.mdl), a Xmile (.xmile, " + ".xml, .stmx) or a PySD (.py) model file...", stderr) def test_read_model_not_exists(self): @@ -198,8 +198,6 @@ def test_read_vensim_split_model(self): root_dir = os.path.join(_root, "more-tests/split_model") + "/" model_name = "test_split_model" - namespace_filename = "_namespace_" + model_name + ".json" - dependencies_filename = "_dependencies_" + model_name + ".json" subscript_filename = "_subscripts_" + model_name + ".json" modules_filename = "_modules.json" modules_dirname = "modules_" + model_name @@ -209,10 +207,8 @@ def test_read_vensim_split_model(self): out = subprocess.run(split_bash(command), capture_output=True) self.assertEqual(out.returncode, 0) - # check that _namespace and _subscript_dict json files where created - self.assertTrue(os.path.isfile(root_dir + namespace_filename)) + # check that _subscript_dict json file was created self.assertTrue(os.path.isfile(root_dir + subscript_filename)) - self.assertTrue(os.path.isfile(root_dir + dependencies_filename)) # check that the main model file was created self.assertTrue(os.path.isfile(root_dir + model_name + ".py")) @@ -233,9 +229,7 @@ def test_read_vensim_split_model(self): # remove newly created files os.remove(root_dir + model_name + ".py") - os.remove(root_dir + namespace_filename) os.remove(root_dir + subscript_filename) - os.remove(root_dir + dependencies_filename) # remove newly created modules folder shutil.rmtree(root_dir + modules_dirname) @@ -254,9 +248,7 @@ def test_read_vensim_split_model_subviews(self): subview_sep=["."] ) - namespace_filename = "_namespace_" + model_name + ".json" subscript_filename = "_subscripts_" + model_name + ".json" - dependencies_filename = "_dependencies_" + model_name + ".json" modules_dirname = "modules_" + model_name separator = "." @@ -293,9 +285,7 @@ def test_read_vensim_split_model_subviews(self): # remove newly created files os.remove(root_dir + model_name + ".py") - os.remove(root_dir + namespace_filename) os.remove(root_dir + subscript_filename) - os.remove(root_dir + dependencies_filename) # remove newly created modules folder shutil.rmtree(root_dir + modules_dirname) diff --git a/tests/unit_test_external.py b/tests/unit_test_external.py index 78f0aae3..65659037 100644 --- a/tests/unit_test_external.py +++ b/tests/unit_test_external.py @@ -146,20 +146,28 @@ def test_reshape(self): External._reshape test """ from pysd.py_backend.external import External - import pandas as pd reshape = External._reshape + data0d = np.array(5) data1d = np.array([2, 3, 5, 6]) data2d = np.array([[2, 3, 5, 6], [1, 7, 5, 8]]) - series1d = pd.Series(data1d) - df2d = pd.DataFrame(data2d) + float0d = float(data0d) + int0d = int(data0d) + series1d = xr.DataArray(data1d) + df2d = xr.DataArray(data2d) + shapes0d = [(1,), (1, 1)] shapes1d = [(4,), (4, 1, 1), (1, 1, 4), (1, 4, 1)] shapes2d = [(2, 4), (2, 4, 1), (1, 2, 4), (2, 1, 4)] + for shape_i in shapes0d: + self.assertEqual(reshape(data0d, shape_i).shape, shape_i) + self.assertEqual(reshape(float0d, shape_i).shape, shape_i) + self.assertEqual(reshape(int0d, shape_i).shape, shape_i) + for shape_i in shapes1d: self.assertEqual(reshape(data1d, shape_i).shape, shape_i) self.assertEqual(reshape(series1d, shape_i).shape, shape_i) @@ -207,12 +215,12 @@ def test_fill_missing(self): interp = np.array([1., 1., 1., 3., 3.5, 4., 5., 6., 7., 8., 8., 8.]) - ext.interp = "hold backward" + ext.interp = "hold_backward" datac = data.copy() ext._fill_missing(series, datac) self.assertTrue(np.all(hold_back == datac)) - ext.interp = "look forward" + ext.interp = "look_forward" datac = data.copy() ext._fill_missing(series, datac) self.assertTrue(np.all(look_for == datac)) @@ -257,7 +265,7 @@ def test_resolve_file(self): ext._resolve_file(root=root) self.assertIn( - "Indirect reference to file: ?input.xlsx", + "Indirect reference to file: '?input.xlsx'", str(err.exception)) @@ -294,6 +302,7 @@ def test_data_interp_h1d_1(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -326,6 +335,7 @@ def test_data_interp_hn1d_1(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -355,6 +365,7 @@ def test_data_interp_h1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -385,6 +396,7 @@ def test_data_interp_v1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -415,6 +427,7 @@ def test_data_interp_hn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -445,6 +458,7 @@ def test_data_interp_vn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -456,7 +470,7 @@ def test_data_interp_vn1d(self): def test_data_forward_h1d(self): """ - ExtData test for 1d horizontal series look forward + ExtData test for 1d horizontal series look_forward """ import pysd @@ -465,7 +479,7 @@ def test_data_forward_h1d(self): time_row_or_col = "4" cell = "C5" coords = {} - interp = "look forward" + interp = "look_forward" py_name = "test_data_forward_h1d" data = pysd.external.ExtData(file_name=file_name, @@ -475,6 +489,7 @@ def test_data_forward_h1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -486,7 +501,7 @@ def test_data_forward_h1d(self): def test_data_forward_v1d(self): """ - ExtData test for 1d vertical series look forward + ExtData test for 1d vertical series look_forward """ import pysd @@ -495,7 +510,7 @@ def test_data_forward_v1d(self): time_row_or_col = "B" cell = "C5" coords = {} - interp = "look forward" + interp = "look_forward" py_name = "test_data_forward_v1d" data = pysd.external.ExtData(file_name=file_name, @@ -505,6 +520,7 @@ def test_data_forward_v1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -516,7 +532,7 @@ def test_data_forward_v1d(self): def test_data_forward_hn1d(self): """ - ExtData test for 1d horizontal series look forward by cell range names + ExtData test for 1d horizontal series look_forward by cell range names """ import pysd @@ -525,7 +541,7 @@ def test_data_forward_hn1d(self): time_row_or_col = "time" cell = "data_1d" coords = {} - interp = "look forward" + interp = "look_forward" py_name = "test_data_forward_hn1d" data = pysd.external.ExtData(file_name=file_name, @@ -535,6 +551,7 @@ def test_data_forward_hn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -546,7 +563,7 @@ def test_data_forward_hn1d(self): def test_data_forward_vn1d(self): """ - ExtData test for 1d vertical series look forward by cell range names + ExtData test for 1d vertical series look_forward by cell range names """ import pysd @@ -555,7 +572,7 @@ def test_data_forward_vn1d(self): time_row_or_col = "time" cell = "data_1d" coords = {} - interp = "look forward" + interp = "look_forward" py_name = "test_data_forward_vn1d" data = pysd.external.ExtData(file_name=file_name, @@ -565,6 +582,7 @@ def test_data_forward_vn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -576,7 +594,7 @@ def test_data_forward_vn1d(self): def test_data_backward_h1d(self): """ - ExtData test for 1d horizontal series hold backward + ExtData test for 1d horizontal series hold_backward """ import pysd @@ -585,7 +603,7 @@ def test_data_backward_h1d(self): time_row_or_col = "4" cell = "C5" coords = {} - interp = "hold backward" + interp = "hold_backward" py_name = "test_data_backward_h1d" data = pysd.external.ExtData(file_name=file_name, @@ -595,6 +613,7 @@ def test_data_backward_h1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -606,7 +625,7 @@ def test_data_backward_h1d(self): def test_data_backward_v1d(self): """ - ExtData test for 1d vertical series hold backward by cell range names + ExtData test for 1d vertical series hold_backward by cell range names """ import pysd @@ -615,7 +634,7 @@ def test_data_backward_v1d(self): time_row_or_col = "B" cell = "C5" coords = {} - interp = "hold backward" + interp = "hold_backward" py_name = "test_data_backward_v1d" data = pysd.external.ExtData(file_name=file_name, @@ -625,6 +644,7 @@ def test_data_backward_v1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -636,7 +656,7 @@ def test_data_backward_v1d(self): def test_data_backward_hn1d(self): """ - ExtData test for 1d horizontal series hold backward by cell range names + ExtData test for 1d horizontal series hold_backward by cell range names """ import pysd @@ -645,7 +665,7 @@ def test_data_backward_hn1d(self): time_row_or_col = "time" cell = "data_1d" coords = {} - interp = "hold backward" + interp = "hold_backward" py_name = "test_data_backward_hn1d" data = pysd.external.ExtData(file_name=file_name, @@ -655,6 +675,7 @@ def test_data_backward_hn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -666,7 +687,7 @@ def test_data_backward_hn1d(self): def test_data_backward_vn1d(self): """ - ExtData test for 1d vertical series hold backward by cell range names + ExtData test for 1d vertical series hold_backward by cell range names """ import pysd @@ -675,7 +696,7 @@ def test_data_backward_vn1d(self): time_row_or_col = "time" cell = "data_1d" coords = {} - interp = "hold backward" + interp = "hold_backward" py_name = "test_data_backward_vn1d" data = pysd.external.ExtData(file_name=file_name, @@ -685,6 +706,7 @@ def test_data_backward_vn1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -715,6 +737,7 @@ def test_data_interp_vn2d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -726,7 +749,7 @@ def test_data_interp_vn2d(self): def test_data_forward_hn2d(self): """ - ExtData test for 2d vertical series look forward by cell range names + ExtData test for 2d vertical series look_forward by cell range names """ import pysd @@ -735,7 +758,7 @@ def test_data_forward_hn2d(self): time_row_or_col = "time" cell = "data_2d" coords = {'ABC': ['A', 'B', 'C']} - interp = "look forward" + interp = "look_forward" py_name = "test_data_forward_hn2d" data = pysd.external.ExtData(file_name=file_name, @@ -745,6 +768,7 @@ def test_data_forward_hn2d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -756,7 +780,7 @@ def test_data_forward_hn2d(self): def test_data_backward_v2d(self): """ - ExtData test for 2d vertical series hold backward + ExtData test for 2d vertical series hold_backward """ import pysd @@ -765,7 +789,7 @@ def test_data_backward_v2d(self): time_row_or_col = "B" cell = "C5" coords = {'ABC': ['A', 'B', 'C']} - interp = "hold backward" + interp = "hold_backward" py_name = "test_data_backward_v2d" data = pysd.external.ExtData(file_name=file_name, @@ -775,6 +799,7 @@ def test_data_backward_v2d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -798,6 +823,7 @@ def test_data_interp_h3d(self): cell_2 = "C8" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} interp = None py_name = "test_data_interp_h3d" @@ -808,6 +834,7 @@ def test_data_interp_h3d(self): cell=cell_1, coords=coords_1, interp=interp, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -827,7 +854,7 @@ def test_data_interp_h3d(self): def test_data_forward_v3d(self): """ - ExtData test for 3d vertical series look forward + ExtData test for 3d vertical series look_forward """ import pysd @@ -838,7 +865,8 @@ def test_data_forward_v3d(self): cell_2 = "F5" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} - interp = "look forward" + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} + interp = "look_forward" py_name = "test_data_forward_v3d" data = pysd.external.ExtData(file_name=file_name, @@ -848,6 +876,7 @@ def test_data_forward_v3d(self): cell=cell_1, coords=coords_1, interp=interp, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -867,7 +896,7 @@ def test_data_forward_v3d(self): def test_data_backward_hn3d(self): """ - ExtData test for 3d horizontal series hold backward by cellrange names + ExtData test for 3d horizontal series hold_backward by cellrange names """ import pysd @@ -878,7 +907,8 @@ def test_data_backward_hn3d(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} - interp = "hold backward" + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} + interp = "hold_backward" py_name = "test_data_backward_hn3d" data = pysd.external.ExtData(file_name=file_name, @@ -888,6 +918,7 @@ def test_data_backward_hn3d(self): cell=cell_1, coords=coords_1, interp=interp, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -926,6 +957,7 @@ def test_data_raw_h1d(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -967,6 +999,7 @@ def test_lookup_h1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -995,6 +1028,7 @@ def test_lookup_v1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1023,6 +1057,7 @@ def test_lookup_hn1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1051,6 +1086,7 @@ def test_lookup_vn1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1079,6 +1115,7 @@ def test_lookup_h2d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1102,6 +1139,7 @@ def test_lookup_vn3d(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_vn3d" data = pysd.external.ExtLookup(file_name=file_name, @@ -1110,6 +1148,7 @@ def test_lookup_vn3d(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1140,6 +1179,7 @@ def test_lookup_vn3d_shape0(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_vn3d_shape0" data = pysd.external.ExtLookup(file_name=file_name, @@ -1148,6 +1188,7 @@ def test_lookup_vn3d_shape0(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1184,17 +1225,19 @@ def test_lookup_vn2d_xarray(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=coords_1, py_name=py_name) data.initialize() - all_smaller = xr.DataArray([-1, -10], {'XY': ['X', 'Y']}, ['XY']) - all_bigger = xr.DataArray([9, 20, 30], {'ABC': ['A', 'B', 'C']}, - ['ABC']) - all_inside = xr.DataArray([3.5, 5.5], {'XY': ['X', 'Y']}, ['XY']) - mixed = xr.DataArray([1.5, 20, -30], {'ABC': ['A', 'B', 'C']}, ['ABC']) + coords_2 = {'XY': ['X', 'Y']} + + all_smaller = xr.DataArray([-1, -10], coords_2, ['XY']) + all_bigger = xr.DataArray([9, 20, 30], coords_1, ['ABC']) + all_inside = xr.DataArray([3.5, 5.5], coords_2, ['XY']) + mixed = xr.DataArray([1.5, 20, -30], coords_1, ['ABC']) full = xr.DataArray([[1.5, -30], [-10, 2.5], [4., 5.]], - {'ABC': ['A', 'B', 'C'], 'XY': ['X', 'Y']}, + {**coords_1, **coords_2}, ['ABC', 'XY']) all_smaller_out = data.data[0].reset_coords('lookup_dim', drop=True)\ @@ -1203,25 +1246,30 @@ def test_lookup_vn2d_xarray(self): all_inside_out = xr.DataArray([[0.5, -1], [-1, -0.5], [-0.75, 0]], - {'ABC': ['A', 'B', 'C'], - 'XY': ['X', 'Y']}, + {**coords_1, **coords_2}, ['ABC', 'XY']) - mixed_out = xr.DataArray([0.5, 0, 1], - {'ABC': ['A', 'B', 'C']}, - ['ABC']) - full_out = xr.DataArray([[0.5, 0], - [0, 0], - [-0.5, 0]], - {'ABC': ['A', 'B', 'C'], 'XY': ['X', 'Y']}, - ['ABC', 'XY']) + mixed_out = xr.DataArray([0.5, 0, 1], coords_1, ['ABC']) + full_out = xr.DataArray([[0.5, 0, -0.5], + [0, 0, 0]], + {**coords_2, **coords_1}, + ['XY', 'ABC']) with warnings.catch_warnings(): warnings.simplefilter("ignore") - self.assertTrue(data(all_smaller).equals(all_smaller_out)) - self.assertTrue(data(all_bigger).equals(all_bigger_out)) - self.assertTrue(data(all_inside).equals(all_inside_out)) - self.assertTrue(data(mixed).equals(mixed_out)) - self.assertTrue(data(full).equals(full_out)) + self.assertTrue( + data( + all_smaller, {**coords_1, **coords_2} + ).equals(all_smaller_out)) + self.assertTrue( + data(all_bigger, coords_1).equals(all_bigger_out)) + self.assertTrue( + data( + all_inside, {**coords_1, **coords_2} + ).equals(all_inside_out)) + self.assertTrue( + data(mixed, coords_1).equals(mixed_out)) + self.assertTrue( + data(full, {**coords_2, **coords_1}).equals(full_out)) def test_lookup_vn3d_xarray(self): """ @@ -1237,6 +1285,7 @@ def test_lookup_vn3d_xarray(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_vn3d_xarray" data = pysd.external.ExtLookup(file_name=file_name, @@ -1245,6 +1294,7 @@ def test_lookup_vn3d_xarray(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1283,11 +1333,16 @@ def test_lookup_vn3d_xarray(self): with warnings.catch_warnings(): warnings.simplefilter("ignore") - self.assertTrue(data(all_smaller).equals(all_smaller_out)) - self.assertTrue(data(all_bigger).equals(all_bigger_out)) - self.assertTrue(data(all_inside).equals(all_inside_out)) - self.assertTrue(data(mixed).equals(mixed_out)) - self.assertTrue(data(full).equals(full_out)) + self.assertTrue( + data(all_smaller, final_coords).equals(all_smaller_out)) + self.assertTrue( + data(all_bigger, final_coords).equals(all_bigger_out)) + self.assertTrue( + data(all_inside, final_coords).equals(all_inside_out)) + self.assertTrue( + data(mixed, final_coords).equals(mixed_out)) + self.assertTrue( + data(full, final_coords).equals(full_out)) class TestConstant(unittest.TestCase): @@ -1315,6 +1370,7 @@ def test_constant_0d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data2 = pysd.external.ExtConstant(file_name=file_name, @@ -1322,6 +1378,7 @@ def test_constant_0d(self): root=_root, cell=cell2, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() data2.initialize() @@ -1347,6 +1404,7 @@ def test_constant_n0d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data2 = pysd.external.ExtConstant(file_name=file_name, @@ -1354,6 +1412,7 @@ def test_constant_n0d(self): root=_root, cell=cell2, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() data2.initialize() @@ -1378,6 +1437,7 @@ def test_constant_h1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1400,6 +1460,7 @@ def test_constant_v1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1422,6 +1483,7 @@ def test_constant_hn1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1444,6 +1506,7 @@ def test_constant_vn1d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1466,6 +1529,7 @@ def test_constant_h2d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1488,6 +1552,7 @@ def test_constant_v2d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1510,6 +1575,7 @@ def test_constant_hn2d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1532,6 +1598,7 @@ def test_constant_vn2d(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -1553,6 +1620,9 @@ def test_constant_h3d(self): coords2 = {'ABC': ['A', 'B', 'C'], 'XY': ['Y'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'ABC': ['A', 'B', 'C'], + 'XY': ['X', 'Y'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_h3d" data = pysd.external.ExtConstant(file_name=file_name, @@ -1560,6 +1630,7 @@ def test_constant_h3d(self): root=_root, cell=cell, coords=coords, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1587,6 +1658,9 @@ def test_constant_v3d(self): coords2 = {'ABC': ['A', 'B', 'C'], 'XY': ['Y'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'ABC': ['A', 'B', 'C'], + 'XY': ['X', 'Y'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_v3d" data = pysd.external.ExtConstant(file_name=file_name, @@ -1594,6 +1668,7 @@ def test_constant_v3d(self): root=_root, cell=cell, coords=coords, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1621,6 +1696,9 @@ def test_constant_hn3d(self): coords2 = {'ABC': ['A', 'B', 'C'], 'XY': ['Y'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'ABC': ['A', 'B', 'C'], + 'XY': ['X', 'Y'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_hn3d" data = pysd.external.ExtConstant(file_name=file_name, @@ -1628,6 +1706,7 @@ def test_constant_hn3d(self): root=_root, cell=cell, coords=coords, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1655,6 +1734,9 @@ def test_constant_vn3d(self): coords2 = {'ABC': ['A', 'B', 'C'], 'XY': ['Y'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'ABC': ['A', 'B', 'C'], + 'XY': ['X', 'Y'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_vn2d" data = pysd.external.ExtConstant(file_name=file_name, @@ -1662,6 +1744,7 @@ def test_constant_vn3d(self): root=_root, cell=cell, coords=coords, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -1752,6 +1835,7 @@ def test_not_implemented_file(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(NotImplementedError): @@ -1778,6 +1862,7 @@ def test_non_existent_file(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(FileNotFoundError): @@ -1804,6 +1889,7 @@ def test_non_existent_sheet_pyxl(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -1830,6 +1916,7 @@ def test_non_existent_cellrange_name_pyxl(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(AttributeError): @@ -1852,6 +1939,7 @@ def test_non_existent_cellrange_name_in_sheet_pyxl(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) with self.assertRaises(AttributeError): @@ -1883,6 +1971,7 @@ def test_data_interp_h1dm_row(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -1916,6 +2005,7 @@ def test_data_interp_h1dm_row2(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -1951,6 +2041,7 @@ def test_data_interp_h1dm(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -1995,6 +2086,7 @@ def test_data_interp_h1dm_ignore(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -2038,6 +2130,7 @@ def test_data_interp_h1dm_raise(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2067,6 +2160,7 @@ def test_data_interp_v1dm(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -2111,6 +2205,7 @@ def test_data_interp_v1dm_ignore(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -2154,6 +2249,7 @@ def test_data_interp_v1dm_raise(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2183,6 +2279,7 @@ def test_data_interp_hn1dm(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -2227,6 +2324,7 @@ def test_data_interp_hn1dm_ignore(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with warnings.catch_warnings(record=True) as ws: @@ -2270,6 +2368,7 @@ def test_data_interp_hn1dm_raise(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2289,6 +2388,7 @@ def test_data_interp_hn3dmd(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} interp = "interpolate" py_name = "test_data_interp_hn3dmd" @@ -2301,6 +2401,7 @@ def test_data_interp_hn3dmd(self): cell=cell_1, interp=interp, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2348,6 +2449,7 @@ def test_data_interp_hn3dmd_raw(self): cell_2 = "data_2db" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} interp = "raw" py_name = "test_data_interp_hn3dmd_raw" @@ -2360,6 +2462,7 @@ def test_data_interp_hn3dmd_raw(self): cell=cell_1, interp=interp, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2395,6 +2498,7 @@ def test_lookup_hn3dmd_raise(self): cell_2 = "C19" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_hn3dmd_raise" pysd.external.External.missing = "raise" @@ -2404,6 +2508,7 @@ def test_lookup_hn3dmd_raise(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2429,6 +2534,7 @@ def test_lookup_hn3dmd_ignore(self): cell_2 = "C19" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_hn3dmd_ignore" pysd.external.External.missing = "ignore" @@ -2438,6 +2544,7 @@ def test_lookup_hn3dmd_ignore(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2479,6 +2586,8 @@ def test_constant_h3dm(self): 'val': [0, 1, 2, 3, 5, 6, 7, 8]} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_h3dm" pysd.external.External.missing = "warning" @@ -2487,6 +2596,7 @@ def test_constant_h3dm(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2518,6 +2628,8 @@ def test_constant_h3dm_ignore(self): 'val': [0, 1, 2, 3, 5, 6, 7, 8]} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_h3dm_ignore" pysd.external.External.missing = "ignore" @@ -2526,6 +2638,7 @@ def test_constant_h3dm_ignore(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2553,6 +2666,8 @@ def test_constant_h3dm_raise(self): 'val': [0, 1, 2, 3, 5, 6, 7, 8]} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_h3dm_raise" pysd.external.External.missing = "raise" @@ -2561,6 +2676,7 @@ def test_constant_h3dm_raise(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2586,6 +2702,8 @@ def test_constant_hn3dm_raise(self): 'val': [0, 1, 2, 3, 5, 6, 7, 8]} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_hn3dm_raise" pysd.external.External.missing = "raise" @@ -2594,6 +2712,7 @@ def test_constant_hn3dm_raise(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -2625,6 +2744,7 @@ def test_data_interp_h1d0(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2651,6 +2771,7 @@ def test_data_interp_v1d0(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2679,6 +2800,7 @@ def test_data_interp_hn1d0(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2706,6 +2828,7 @@ def test_data_interp_hn1dt(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2733,6 +2856,7 @@ def test_data_interp_hns(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2760,6 +2884,7 @@ def test_data_interp_vnss(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2788,6 +2913,7 @@ def test_data_interp_hnnwd(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError) as err: @@ -2816,6 +2942,7 @@ def test_data_raw_hnnm(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -2839,6 +2966,7 @@ def test_data_raw_hnnm(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -2874,6 +3002,7 @@ def test_data_h3d_interpnv(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) def test_data_h3d_interp(self): @@ -2889,8 +3018,9 @@ def test_data_h3d_interp(self): cell_2 = "C8" coords_1 = {'ABC': ['A', 'B', 'C'], 'XY': ['X']} coords_2 = {'ABC': ['A', 'B', 'C'], 'XY': ['Y']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} interp = None - interp2 = "look forward" + interp2 = "look_forward" py_name = "test_data_h3d_interp" data = pysd.external.ExtData(file_name=file_name, @@ -2900,6 +3030,7 @@ def test_data_h3d_interp(self): cell=cell_1, coords=coords_1, interp=interp, + final_coords=final_coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2923,6 +3054,7 @@ def test_data_h3d_add(self): cell_2 = "C8" coords_1 = {'ABC': ['A', 'B', 'C'], 'XY': ['X']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} interp = None py_name = "test_data_h3d_add" @@ -2933,6 +3065,7 @@ def test_data_h3d_add(self): cell=cell_1, coords=coords_1, interp=interp, + final_coords=final_coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2956,6 +3089,7 @@ def test_lookup_h3d_add(self): cell_2 = "C8" coords_1 = {'ABC': ['A', 'B', 'C'], 'XY': ['X']} coords_2 = {'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_h3d_add" data = pysd.external.ExtLookup(file_name=file_name, @@ -2964,6 +3098,7 @@ def test_lookup_h3d_add(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) with self.assertRaises(ValueError): @@ -2989,6 +3124,8 @@ def test_constant_h3d_add(self): coords2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val2': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_h3d_add" data = pysd.external.ExtConstant(file_name=file_name, @@ -2996,6 +3133,7 @@ def test_constant_h3d_add(self): root=_root, cell=cell, coords=coords, + final_coords=final_coords, py_name=py_name) with self.assertRaises(ValueError): @@ -3022,6 +3160,7 @@ def test_constant_hns(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) with self.assertRaises(ValueError): @@ -3048,6 +3187,7 @@ def text_openpyxl_str(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) expected = xr.DataArray( @@ -3067,6 +3207,7 @@ def text_openpyxl_str(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) data.initialize() @@ -3095,6 +3236,8 @@ def test_constant_hn3dm_keep(self): 'val': [0, 1, 2, 3, 5, 6, 7, 8]} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C'], 'val': [0, 1, 2, 3, 5, 6, 7, 8]} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C'], + 'val': [0, 1, 2, 3, 5, 6, 7, 8]} py_name = "test_constant_hn3dm_raise" pysd.external.External.missing = "keep" @@ -3114,6 +3257,7 @@ def test_constant_hn3dm_keep(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -3139,6 +3283,7 @@ def test_lookup_hn3dmd_keep(self): cell_2 = "C19" coords_1 = {'XY': ['X'], 'ABC': ['A', 'B', 'C']} coords_2 = {'XY': ['Y'], 'ABC': ['A', 'B', 'C']} + final_coords = {'XY': ['X', 'Y'], 'ABC': ['A', 'B', 'C']} py_name = "test_lookup_hn3dmd_ignore" pysd.external.External.missing = "keep" @@ -3169,6 +3314,7 @@ def test_lookup_hn3dmd_keep(self): root=_root, cell=cell_1, coords=coords_1, + final_coords=final_coords, py_name=py_name) data.add(file_name=file_name, @@ -3210,6 +3356,7 @@ def test_data_interp_v1dm_keep(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -3245,6 +3392,7 @@ def test_data_interp_hnnm_keep(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) data.initialize() @@ -3273,6 +3421,7 @@ def test_lookup_data_attr(self): cell=cell, coords=coords, interp=interp, + final_coords=coords, py_name=py_name) datL = pysd.external.ExtLookup(file_name=file_name, @@ -3281,6 +3430,7 @@ def test_lookup_data_attr(self): root=_root, cell=cell, coords=coords, + final_coords=coords, py_name=py_name) datD.initialize() datL.initialize() diff --git a/tests/unit_test_functions.py b/tests/unit_test_functions.py deleted file mode 100644 index 39ec8d0d..00000000 --- a/tests/unit_test_functions.py +++ /dev/null @@ -1,434 +0,0 @@ -import unittest - -import numpy as np -import xarray as xr - - -class TestInputFunctions(unittest.TestCase): - def test_ramp(self): - from pysd.py_backend.functions import ramp - - self.assertEqual(ramp(lambda: 14, .5, 10, 18), 2) - - self.assertEqual(ramp(lambda: 4, .5, 10, 18), 0) - - self.assertEqual(ramp(lambda: 24, .5, 10, 18), 4) - - self.assertEqual(ramp(lambda: 24, .5, 10, -1), 7) - - def test_step(self): - from pysd.py_backend.functions import step - - self.assertEqual(step(lambda: 5, 1, 10), 0) - - self.assertEqual(step(lambda: 15, 1, 10), 1) - - self.assertEqual(step(lambda: 10, 1, 10), 1) - - def test_pulse(self): - from pysd.py_backend.functions import pulse - - self.assertEqual(pulse(lambda: 0, 1, 3), 0) - - self.assertEqual(pulse(lambda: 1, 1, 3), 1) - - self.assertEqual(pulse(lambda: 2, 1, 3), 1) - - self.assertEqual(pulse(lambda: 4, 1, 3), 0) - - self.assertEqual(pulse(lambda: 5, 1, 3), 0) - - def test_pulse_chain(self): - from pysd.py_backend.functions import pulse_train - # before train starts - self.assertEqual(pulse_train(lambda: 0, 1, 3, 5, 12), 0) - # on train start - self.assertEqual(pulse_train(lambda: 1, 1, 3, 5, 12), 1) - # within first pulse - self.assertEqual(pulse_train(lambda: 2, 1, 3, 5, 12), 1) - # end of first pulse - self.assertEqual(pulse_train(lambda: 4, 1, 3, 5, 12), 0) - # after first pulse before second - self.assertEqual(pulse_train(lambda: 5, 1, 3, 5, 12), 0) - # on start of second pulse - self.assertEqual(pulse_train(lambda: 6, 1, 3, 5, 12), 1) - # within second pulse - self.assertEqual(pulse_train(lambda: 7, 1, 3, 5, 12), 1) - # after second pulse - self.assertEqual(pulse_train(lambda: 10, 1, 3, 5, 12), 0) - # on third pulse - self.assertEqual(pulse_train(lambda: 11, 1, 3, 5, 12), 1) - # on train end - self.assertEqual(pulse_train(lambda: 12, 1, 3, 5, 12), 0) - # after train - self.assertEqual(pulse_train(lambda: 15, 1, 3, 5, 13), 0) - - def test_pulse_magnitude(self): - from pysd.py_backend.functions import pulse_magnitude - from pysd.py_backend.statefuls import Time - - # Pulse function with repeat time - # before first impulse - t = Time() - t.set_control_vars(initial_time=0, time_step=1) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 0) - # first impulse - t.update(2) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 10) - # after first impulse and before second - t.update(4) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 0) - # second impulse - t.update(7) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 10) - # after second and before third impulse - t.update(9) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 0) - # third impulse - t.update(12) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 10) - # after third impulse - t.update(14) - self.assertEqual(pulse_magnitude(t, 10, 2, 5), 0) - - # Pulse function without repeat time - # before first impulse - t = Time() - t.set_control_vars(initial_time=0, time_step=1) - self.assertEqual(pulse_magnitude(t, 10, 2), 0) - # first impulse - t.update(2) - self.assertEqual(pulse_magnitude(t, 10, 2), 10) - # after first impulse and before second - t.update(4) - self.assertEqual(pulse_magnitude(t, 10, 2), 0) - # second impulse - t.update(7) - self.assertEqual(pulse_magnitude(t, 10, 2), 0) - # after second and before third impulse - t.update(9) - self.assertEqual(pulse_magnitude(t, 10, 2), 0) - - def test_xidz(self): - from pysd.py_backend.functions import xidz - self.assertEqual(xidz(1, -0.00000001, 5), 5) - self.assertEqual(xidz(1, 0, 5), 5) - self.assertEqual(xidz(1, 8, 5), 0.125) - - def test_zidz(self): - from pysd.py_backend.functions import zidz - self.assertEqual(zidz(1, -0.00000001), 0) - self.assertEqual(zidz(1, 0), 0) - self.assertEqual(zidz(1, 8), 0.125) - - -class TestStatsFunctions(unittest.TestCase): - def test_bounded_normal(self): - from pysd.py_backend.functions import bounded_normal - min_val = -4 - max_val = .2 - mean = -1 - std = .05 - seed = 1 - results = np.array( - [bounded_normal(min_val, max_val, mean, std, seed) - for _ in range(1000)]) - - self.assertGreaterEqual(results.min(), min_val) - self.assertLessEqual(results.max(), max_val) - self.assertAlmostEqual(results.mean(), mean, delta=std) - self.assertAlmostEqual(results.std(), std, delta=std) - self.assertGreater(len(np.unique(results)), 100) - - -class TestLogicFunctions(unittest.TestCase): - def test_if_then_else_basic(self): - from pysd.py_backend.functions import if_then_else - self.assertEqual(if_then_else(True, lambda: 1, lambda: 0), 1) - self.assertEqual(if_then_else(False, lambda: 1, lambda: 0), 0) - - # Ensure lazzy evaluation - self.assertEqual(if_then_else(True, lambda: 1, lambda: 1/0), 1) - self.assertEqual(if_then_else(False, lambda: 1/0, lambda: 0), 0) - - with self.assertRaises(ZeroDivisionError): - if_then_else(True, lambda: 1/0, lambda: 0) - with self.assertRaises(ZeroDivisionError): - if_then_else(False, lambda: 1, lambda: 1/0) - - def test_if_then_else_with_subscripted(self): - # this test only test the lazzy evaluation and basics - # subscripted_if_then_else test all the possibilities - - from pysd.py_backend.functions import if_then_else - - coords = {'dim1': [0, 1], 'dim2': [0, 1]} - dims = list(coords) - - xr_true = xr.DataArray([[True, True], [True, True]], coords, dims) - xr_false = xr.DataArray([[False, False], [False, False]], coords, dims) - xr_mixed = xr.DataArray([[True, False], [False, True]], coords, dims) - - out_mixed = xr.DataArray([[1, 0], [0, 1]], coords, dims) - - self.assertEqual(if_then_else(xr_true, lambda: 1, lambda: 0), 1) - self.assertEqual(if_then_else(xr_false, lambda: 1, lambda: 0), 0) - self.assertTrue( - if_then_else(xr_mixed, lambda: 1, lambda: 0).equals(out_mixed)) - - # Ensure lazzy evaluation - self.assertEqual(if_then_else(xr_true, lambda: 1, lambda: 1/0), 1) - self.assertEqual(if_then_else(xr_false, lambda: 1/0, lambda: 0), 0) - - with self.assertRaises(ZeroDivisionError): - if_then_else(xr_true, lambda: 1/0, lambda: 0) - with self.assertRaises(ZeroDivisionError): - if_then_else(xr_false, lambda: 1, lambda: 1/0) - with self.assertRaises(ZeroDivisionError): - if_then_else(xr_mixed, lambda: 1/0, lambda: 0) - with self.assertRaises(ZeroDivisionError): - if_then_else(xr_mixed, lambda: 1, lambda: 1/0) - - -class TestLookup(unittest.TestCase): - def test_lookup(self): - from pysd.py_backend.functions import lookup - - xpts = [0, 1, 2, 3, 5, 6, 7, 8] - ypts = [0, 0, 1, 1, -1, -1, 0, 0] - - for x, y in zip(xpts, ypts): - self.assertEqual( - y, - lookup(x, xpts, ypts), - "Wrong result at X=" + str(x)) - - def test_lookup_extrapolation_inbounds(self): - from pysd.py_backend.functions import lookup_extrapolation - - xpts = [0, 1, 2, 3, 5, 6, 7, 8] - ypts = [0, 0, 1, 1, -1, -1, 0, 0] - - expected_xpts = np.arange(-0.5, 8.6, 0.5) - expected_ypts = [ - 0, - 0, 0, - 0, 0.5, - 1, 1, - 1, 0.5, 0, -0.5, -1, - -1, -1, -0.5, - 0, 0, 0, 0 - ] - - for x, y in zip(expected_xpts, expected_ypts): - self.assertEqual( - y, - lookup_extrapolation(x, xpts, ypts), - "Wrong result at X=" + str(x)) - - def test_lookup_extrapolation_two_points(self): - from pysd.py_backend.functions import lookup_extrapolation - - xpts = [0, 1] - ypts = [0, 1] - - expected_xpts = np.arange(-0.5, 1.6, 0.5) - expected_ypts = [-0.5, 0.0, 0.5, 1.0, 1.5] - - for x, y in zip(expected_xpts, expected_ypts): - self.assertEqual( - y, - lookup_extrapolation(x, xpts, ypts), - "Wrong result at X=" + str(x)) - - def test_lookup_extrapolation_outbounds(self): - from pysd.py_backend.functions import lookup_extrapolation - - xpts = [0, 1, 2, 3] - ypts = [0, 1, 1, 0] - - expected_xpts = np.arange(-0.5, 3.6, 0.5) - expected_ypts = [ - -0.5, - 0.0, 0.5, 1.0, - 1.0, 1.0, - 0.5, 0, - -0.5 - ] - - for x, y in zip(expected_xpts, expected_ypts): - self.assertEqual( - y, - lookup_extrapolation(x, xpts, ypts), - "Wrong result at X=" + str(x)) - - def test_lookup_discrete(self): - from pysd.py_backend.functions import lookup_discrete - - xpts = [0, 1, 2, 3, 5, 6, 7, 8] - ypts = [0, 0, 1, 1, -1, -1, 0, 0] - - expected_xpts = np.arange(-0.5, 8.6, 0.5) - expected_ypts = [ - 0, 0, 0, 0, 0, - 1, 1, 1, 1, 1, 1, - -1, -1, -1, -1, - 0, 0, 0, 0 - ] - - for x, y in zip(expected_xpts, expected_ypts): - self.assertEqual( - y, - lookup_discrete(x, xpts, ypts), - "Wrong result at X=" + str(x)) - - -class TestFunctions(unittest.TestCase): - - def test_sum(self): - """ - Test for sum function - """ - from pysd.py_backend.functions import sum - - coords = {'d1': [9, 1], 'd2': [2, 4]} - coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} - dims = ['d1', 'd2'] - - data = xr.DataArray([[1, 2], [3, 4]], coords, dims) - - self.assertTrue(sum( - data, - dim=['d1']).equals(xr.DataArray([4, 6], coords_d2, ['d2']))) - self.assertTrue(sum( - data, - dim=['d2']).equals(xr.DataArray([3, 7], coords_d1, ['d1']))) - self.assertEqual(sum(data, dim=['d1', 'd2']), 10) - self.assertEqual(sum(data), 10) - - def test_prod(self): - """ - Test for sum function - """ - from pysd.py_backend.functions import prod - - coords = {'d1': [9, 1], 'd2': [2, 4]} - coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} - dims = ['d1', 'd2'] - - data = xr.DataArray([[1, 2], [3, 4]], coords, dims) - - self.assertTrue(prod( - data, - dim=['d1']).equals(xr.DataArray([3, 8], coords_d2, ['d2']))) - self.assertTrue(prod( - data, - dim=['d2']).equals(xr.DataArray([2, 12], coords_d1, ['d1']))) - self.assertEqual(prod(data, dim=['d1', 'd2']), 24) - self.assertEqual(prod(data), 24) - - def test_vmin(self): - """ - Test for vmin function - """ - from pysd.py_backend.functions import vmin - - coords = {'d1': [9, 1], 'd2': [2, 4]} - coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} - dims = ['d1', 'd2'] - - data = xr.DataArray([[1, 2], [3, 4]], coords, dims) - - self.assertTrue(vmin( - data, - dim=['d1']).equals(xr.DataArray([1, 2], coords_d2, ['d2']))) - self.assertTrue(vmin( - data, - dim=['d2']).equals(xr.DataArray([1, 3], coords_d1, ['d1']))) - self.assertEqual(vmin(data, dim=['d1', 'd2']), 1) - self.assertEqual(vmin(data), 1) - - def test_vmax(self): - """ - Test for vmax function - """ - from pysd.py_backend.functions import vmax - - coords = {'d1': [9, 1], 'd2': [2, 4]} - coords_d1, coords_d2 = {'d1': [9, 1]}, {'d2': [2, 4]} - dims = ['d1', 'd2'] - - data = xr.DataArray([[1, 2], [3, 4]], coords, dims) - - self.assertTrue(vmax( - data, - dim=['d1']).equals(xr.DataArray([3, 4], coords_d2, ['d2']))) - self.assertTrue(vmax( - data, - dim=['d2']).equals(xr.DataArray([2, 4], coords_d1, ['d1']))) - self.assertEqual(vmax(data, dim=['d1', 'd2']), 4) - self.assertEqual(vmax(data), 4) - - def test_invert_matrix(self): - """ - Test for invert_matrix function - """ - from pysd.py_backend.functions import invert_matrix - - coords1 = {'d1': ['a', 'b'], 'd2': ['a', 'b']} - coords2 = {'d0': ['1', '2'], 'd1': ['a', 'b'], 'd2': ['a', 'b']} - coords3 = {'d0': ['1', '2'], - 'd1': ['a', 'b', 'c'], - 'd2': ['a', 'b', 'c']} - - data1 = xr.DataArray([[1, 2], [3, 4]], coords1, ['d1', 'd2']) - data2 = xr.DataArray([[[1, 2], [3, 4]], [[-1, 2], [5, 4]]], - coords2, - ['d0', 'd1', 'd2']) - data3 = xr.DataArray([[[1, 2, 3], [3, 7, 2], [3, 4, 6]], - [[-1, 2, 3], [4, 7, 3], [5, 4, 6]]], - coords3, - ['d0', 'd1', 'd2']) - - for data in [data1, data2, data3]: - datai = invert_matrix(data) - self.assertEqual(data.dims, datai.dims) - - if len(data.shape) == 2: - # two dimensions xarrays - self.assertTrue(( - abs(np.dot(data, datai) - np.dot(datai, data)) - < 1e-14 - ).all()) - self.assertTrue(( - abs(np.dot(data, datai) - np.identity(data.shape[-1])) - < 1e-14 - ).all()) - else: - # three dimensions xarrays - for i in range(data.shape[0]): - self.assertTrue(( - abs(np.dot(data[i], datai[i]) - - np.dot(datai[i], data[i])) - < 1e-14 - ).all()) - self.assertTrue(( - abs(np.dot(data[i], datai[i]) - - np.identity(data.shape[-1])) - < 1e-14 - ).all()) - - def test_incomplete(self): - from pysd.py_backend.functions import incomplete - from warnings import catch_warnings - - with catch_warnings(record=True) as w: - incomplete() - self.assertEqual(len(w), 1) - self.assertTrue('Call to undefined function' in str(w[-1].message)) - - def test_not_implemented_function(self): - from pysd.py_backend.functions import not_implemented_function - - with self.assertRaises(NotImplementedError): - not_implemented_function("NIF") diff --git a/tests/unit_test_pysd.py b/tests/unit_test_pysd.py index f3cafa88..05a68eaa 100644 --- a/tests/unit_test_pysd.py +++ b/tests/unit_test_pysd.py @@ -27,22 +27,6 @@ class TestPySD(unittest.TestCase): - def test_load_different_version_error(self): - # old PySD major version - with self.assertRaises(ImportError): - pysd.load(more_tests.joinpath("version/test_old_version.py")) - - # current PySD major version - pysd.load(more_tests.joinpath("version/test_current_version.py")) - - def test_load_type_error(self): - with self.assertRaises(ImportError): - pysd.load(more_tests.joinpath("type_error/test_type_error.py")) - - def test_read_not_model_vensim(self): - with self.assertRaises(ValueError): - pysd.read_vensim( - more_tests.joinpath("not_vensim/test_not_vensim.txt")) def test_run(self): model = pysd.read_vensim(test_model) @@ -125,6 +109,45 @@ def test_run_return_timestamps(self): with self.assertRaises(TypeError): model.run(return_timestamps=timestamps) + # assert that return_timestamps works with float error + stocks = model.run(time_step=0.1, return_timestamps=0.3) + assert 0.3 in stocks.index + + # assert that return_timestamps works with float error + stocks = model.run( + time_step=0.1, return_timestamps=[0.3, 0.1, 10.5, 0.9]) + assert 0.1 in stocks.index + assert 0.3 in stocks.index + assert 0.9 in stocks.index + assert 10.5 in stocks.index + + # assert one timestamp is not returned because is not multiple of + # the time step + warning_message =\ + "The returning time stamp '%s' seems to not be a multiple "\ + "of the time step. This value will not be saved in the output. "\ + "Please, modify the returning timestamps or the integration "\ + "time step to avoid this." + # assert that return_timestamps works with float error + with catch_warnings(record=True) as ws: + stocks = model.run( + time_step=0.1, return_timestamps=[0.3, 0.1, 0.55, 0.9]) + assert str(ws[0].message) == warning_message % 0.55 + assert 0.1 in stocks.index + assert 0.3 in stocks.index + assert 0.9 in stocks.index + assert 0.55 not in stocks.index + + with catch_warnings(record=True) as ws: + stocks = model.run( + time_step=0.1, return_timestamps=[0.3, 0.15, 0.55, 0.95]) + for w, value in zip(ws, [0.15, 0.55, 0.95]): + assert str(w.message) == warning_message % value + assert 0.15 not in stocks.index + assert 0.3 in stocks.index + assert 0.95 not in stocks.index + assert 0.55 not in stocks.index + def test_run_return_timestamps_past_final_time(self): """ If the user enters a timestamp that is longer than the euler timeseries that is defined by the normal model file, should @@ -133,6 +156,7 @@ def test_run_return_timestamps_past_final_time(self): model = pysd.read_vensim(test_model) return_timestamps = list(range(0, 100, 10)) stocks = model.run(return_timestamps=return_timestamps) + print(stocks.index) self.assertSequenceEqual(return_timestamps, list(stocks.index)) def test_return_timestamps_with_range(self): @@ -338,7 +362,8 @@ def test_set_subscripted_value_with_constant(self): model = pysd.read_vensim(test_model_subs) model.set_components({"initial_values": 5, "final_time": 10}) - res = model.run(return_columns=["Initial Values"]) + res = model.run( + return_columns=["Initial Values"], flatten_output=False) self.assertTrue(output.equals(res["Initial Values"].iloc[0])) def test_set_subscripted_value_with_partial_xarray(self): @@ -356,7 +381,8 @@ def test_set_subscripted_value_with_partial_xarray(self): model = pysd.read_vensim(test_model_subs) model.set_components({"Initial Values": input_val, "final_time": 10}) - res = model.run(return_columns=["Initial Values"]) + res = model.run( + return_columns=["Initial Values"], flatten_output=False) self.assertTrue(output.equals(res["Initial Values"].iloc[0])) def test_set_subscripted_value_with_xarray(self): @@ -369,9 +395,43 @@ def test_set_subscripted_value_with_xarray(self): model = pysd.read_vensim(test_model_subs) model.set_components({"initial_values": output, "final_time": 10}) - res = model.run(return_columns=["Initial Values"]) + res = model.run( + return_columns=["Initial Values"], flatten_output=False) self.assertTrue(output.equals(res["Initial Values"].iloc[0])) + def test_set_parameter_data(self): + model = pysd.read_vensim(test_model_data) + timeseries = list(range(31)) + series = pd.Series( + index=timeseries, + data=(50+np.random.rand(len(timeseries)).cumsum()) + ) + + with catch_warnings(): + # avoid warnings related to extrapolation + simplefilter("ignore") + model.set_components({"data_backward": 20, "data_forward": 70}) + + out = model.run( + return_columns=["data_backward", "data_forward"], + flatten_output=False) + + for time in out.index: + self.assertTrue((out["data_backward"][time] == 20).all()) + self.assertTrue((out["data_forward"][time] == 70).all()) + + out = model.run( + return_columns=["data_backward", "data_forward"], + final_time=20, time_step=1, saveper=1, + params={"data_forward": 30, "data_backward": series}, + flatten_output=False) + + for time in out.index: + self.assertTrue((out["data_forward"][time] == 30).all()) + self.assertTrue( + (out["data_backward"][time] == series[time]).all() + ) + def test_set_constant_parameter_lookup(self): model = pysd.read_vensim(test_model_look) @@ -430,6 +490,7 @@ def test_set_timeseries_parameter_lookup(self): params={"lookup_1d": temp_timeseries}, return_columns=["lookup_1d_time"], return_timestamps=timeseries, + flatten_output=False ) self.assertTrue((res["lookup_1d_time"] == temp_timeseries).all()) @@ -438,6 +499,7 @@ def test_set_timeseries_parameter_lookup(self): params={"lookup_2d": temp_timeseries}, return_columns=["lookup_2d_time"], return_timestamps=timeseries, + flatten_output=False ) self.assertTrue( @@ -467,6 +529,7 @@ def test_set_timeseries_parameter_lookup(self): params={"lookup_2d": temp_timeseries2}, return_columns=["lookup_2d_time"], return_timestamps=timeseries, + flatten_output=False ) self.assertTrue( @@ -504,6 +567,7 @@ def test_set_subscripted_timeseries_parameter_with_constant(self): params={"initial_values": temp_timeseries, "final_time": 10}, return_columns=["initial_values"], return_timestamps=timeseries, + flatten_output=False ) self.assertTrue( @@ -534,7 +598,8 @@ def test_set_subscripted_timeseries_parameter_with_partial_xarray(self): out_series = [out_b + val for val in val_series] model.set_components({"initial_values": temp_timeseries, "final_time": 10}) - res = model.run(return_columns=["initial_values"]) + res = model.run( + return_columns=["initial_values"], flatten_output=False) self.assertTrue( np.all( [r.equals(t) for r, t in zip(res["initial_values"].values, @@ -562,6 +627,7 @@ def test_set_subscripted_timeseries_parameter_with_xarray(self): params={"initial_values": temp_timeseries, "final_time": 10}, return_columns=["initial_values"], return_timestamps=timeseries, + flatten_output=False ) self.assertTrue( @@ -580,9 +646,7 @@ def test_docs(self): model = pysd.read_vensim(test_model) self.assertIsInstance(str(model), str) # tests string conversion of - # model - - doc = model.doc() + doc = model.doc self.assertIsInstance(doc, pd.DataFrame) self.assertSetEqual( { @@ -594,12 +658,13 @@ def test_docs(self): "Room Temperature", "SAVEPER", "TIME STEP", + "Time" }, set(doc["Real Name"].values), ) self.assertEqual( - doc[doc["Real Name"] == "Heat Loss to Room"]["Unit"].values[0], + doc[doc["Real Name"] == "Heat Loss to Room"]["Units"].values[0], "Degrees Fahrenheit/Minute", ) self.assertEqual( @@ -612,35 +677,20 @@ def test_docs(self): ) self.assertEqual( doc[doc["Real Name"] == "Characteristic Time"]["Type"].values[0], - "constant" + "Constant" ) self.assertEqual( - doc[doc["Real Name"] == "Teacup Temperature"]["Lims"].values[0], - "(32.0, 212.0)", + doc[doc["Real Name"] + == "Characteristic Time"]["Subtype"].values[0], + "Normal" ) - - def test_docs_multiline_eqn(self): - """ Test that the model prints some documentation """ - - path2model = _root.joinpath( - "test-models/tests/multiple_lines_def/" + - "test_multiple_lines_def.mdl") - model = pysd.read_vensim(path2model) - - doc = model.doc() - - self.assertEqual(doc[doc["Real Name"] == "price"]["Unit"].values[0], - "euros/kg") - self.assertEqual(doc[doc["Real Name"] == "price"]["Py Name"].values[0], - "price") self.assertEqual( - doc[doc["Real Name"] == "price"]["Subs"].values[0], "['fruits']" + doc[doc["Real Name"] == "Teacup Temperature"]["Limits"].values[0], + (32.0, 212.0), ) - self.assertEqual(doc[doc["Real Name"] == "price"]["Eqn"].values[0], - "1.2; .; .; .; 1.4") def test_stepwise_cache(self): - from pysd.py_backend.decorators import Cache + from pysd.py_backend.cache import Cache run_history = [] result_history = [] @@ -684,7 +734,7 @@ def downstream(run_hist, res_hist): "up", "down"]) def test_runwise_cache(self): - from pysd.py_backend.decorators import constant_cache + from pysd.py_backend.cache import constant_cache run_history = [] result_history = [] @@ -743,7 +793,7 @@ def test_initialize_order(self): ["_integ_stock_a", "_integ_stock_b"]) self.assertEqual(model.components.stock_b(), 42) self.assertEqual(model.components.stock_a(), 42) - model.components.initial_parameter = lambda: 1 + model.components.initial_par = lambda: 1 model.initialize() self.assertEqual(model.components.stock_b(), 1) self.assertEqual(model.components.stock_a(), 1) @@ -1045,10 +1095,10 @@ def test_get_args(self): self.assertEqual(model.get_args('teacup_temperature'), []) self.assertEqual(model.get_args('_integ_teacup_temperature'), []) - self.assertEqual(model2.get_args('lookup 1d'), ['x']) - self.assertEqual(model2.get_args('lookup_1d'), ['x']) - self.assertEqual(model2.get_args('lookup 2d'), ['x']) - self.assertEqual(model2.get_args('lookup_2d'), ['x']) + self.assertEqual(model2.get_args('lookup 1d'), ['x', 'final_subs']) + self.assertEqual(model2.get_args('lookup_1d'), ['x', 'final_subs']) + self.assertEqual(model2.get_args('lookup 2d'), ['x', 'final_subs']) + self.assertEqual(model2.get_args('lookup_2d'), ['x', 'final_subs']) with self.assertRaises(NameError): model.get_args('not_a_var') @@ -1121,15 +1171,13 @@ def test_get_series_data(self): with self.assertRaises(ValueError) as err: model.get_series_data('Room Temperature') self.assertIn( - "Trying to get the values of a hardcoded lookup/data " - "or other type of variable.", + "Trying to get the values of a constant variable.", err.args[0]) with self.assertRaises(ValueError) as err: model.get_series_data('Teacup Temperature') self.assertIn( - "Trying to get the values of a hardcoded lookup/data " - "or other type of variable.", + "Trying to get the values of a constant variable.", err.args[0]) lookup_exp = xr.DataArray( @@ -1308,19 +1356,6 @@ def test_restart_cache(self): self.assertEqual(new, 345) self.assertNotEqual(old, new) - def test_circular_reference(self): - with self.assertRaises(ValueError) as err: - pysd.load(more_tests.joinpath( - "circular_reference/test_circular_reference.py")) - - self.assertIn("_integ_integ", str(err.exception)) - self.assertIn("_delay_delay", str(err.exception)) - self.assertIn( - "Circular initialization...\n" - + "Not able to initialize the following objects:", - str(err.exception), - ) - def test_not_able_to_update_stateful_object(self): integ = pysd.statefuls.Integ( lambda: xr.DataArray([1, 2], {"Dim": ["A", "B"]}, ["Dim"]), @@ -1371,7 +1406,7 @@ def test_teacup_deps(self): 'saveper': {'time_step': 1}, 'time_step': {} } - self.assertEqual(model.components._dependencies, expected_dep) + self.assertEqual(model.dependencies, expected_dep) def test_multiple_deps(self): from pysd import read_vensim @@ -1382,7 +1417,7 @@ def test_multiple_deps(self): + "test_subscript_individually_defined_stocks2.mdl")) expected_dep = { - "stock_a": {"_integ_stock_a": 2}, + "stock_a": {"_integ_stock_a": 1, "_integ_stock_a_1": 1}, "inflow_a": {"rate_a": 1}, "inflow_b": {"rate_a": 1}, "initial_values": {"initial_values_a": 1, "initial_values_b": 1}, @@ -1394,11 +1429,15 @@ def test_multiple_deps(self): "saveper": {"time_step": 1}, "time_step": {}, "_integ_stock_a": { - "initial": {"initial_values": 2}, - "step": {"inflow_a": 1, "inflow_b": 1} + "initial": {"initial_values": 1}, + "step": {"inflow_a": 1} }, + '_integ_stock_a_1': { + 'initial': {'initial_values': 1}, + 'step': {'inflow_b': 1} + } } - self.assertEqual(model.components._dependencies, expected_dep) + self.assertEqual(model.dependencies, expected_dep) more_tests.joinpath( "subscript_individually_defined_stocks2/" @@ -1418,7 +1457,7 @@ def test_constant_deps(self): "time_step": {}, "saveper": {"time_step": 1} } - self.assertEqual(model.components._dependencies, expected_dep) + self.assertEqual(model.dependencies, expected_dep) for key, value in model.cache_type.items(): if key != "time": diff --git a/tests/unit_test_statefuls.py b/tests/unit_test_statefuls.py index f3eb3b4f..4c78907e 100644 --- a/tests/unit_test_statefuls.py +++ b/tests/unit_test_statefuls.py @@ -1,13 +1,12 @@ import unittest -import os import warnings - +from pathlib import Path import xarray as xr -_root = os.path.dirname(__file__) +_root = Path(__file__).parent -more_tests = os.path.join(_root, "more-tests") +more_tests = _root / "more-tests" class TestStateful(unittest.TestCase): @@ -222,6 +221,7 @@ def input(): frcst = Forecast(forecast_input=input, average_time=lambda: 3, horizon=lambda: 10, + initial_trend=lambda: 0, py_name='forecast') frcst.initialize() @@ -238,11 +238,13 @@ def input(): input_val*(1+(input_val-frcst.state)/(3*frcst.state)*10)) input_val = 7 - init_val = 6 - frcst.initialize(init_val) + init_trend = 6 + + frcst.initialize(init_trend) self.assertEqual( frcst(), - input_val*(1+(input_val-init_val)/(3*init_val)*10)) + input_val * (1+(input_val-input_val/(1+init_trend)) + / (3*input_val/(1+init_trend))*10)) def test_initial(self): from pysd.py_backend.statefuls import Initial @@ -354,9 +356,12 @@ def test_not_initialized_object(self): class TestMacroMethods(unittest.TestCase): def test_get_elements_to_initialize(self): - from pysd.py_backend.statefuls import Macro + from pysd import read_vensim + from pysd.py_backend.model import Macro - macro = Macro(more_tests + "/version/test_current_version.py") + test_model = _root.joinpath("test-models/samples/teacup/teacup.mdl") + read_vensim(test_model) + macro = Macro(test_model.with_suffix(".py")) macro.stateful_initial_dependencies = { "A": {"B", "C"}, diff --git a/tests/unit_test_translation_utils.py b/tests/unit_test_translation_utils.py deleted file mode 100644 index 10a33434..00000000 --- a/tests/unit_test_translation_utils.py +++ /dev/null @@ -1,245 +0,0 @@ -from unittest import TestCase - - -class TestTranslationUtils(TestCase): - - def test_add_entries_underscore(self): - """" - Test for add_entries_undescore - """ - from pysd.translation.utils import add_entries_underscore - - dict1 = {'CD': 10, 'L F': 5} - dict2 = {'a b': 1, 'C': 2, 'L M H': 4} - - dict1b = dict1.copy() - - add_entries_underscore(dict1b) - - self.assertTrue('L_F' in dict1b) - self.assertEqual(dict1b['L F'], dict1b['L_F']) - - add_entries_underscore(dict1, dict2) - - self.assertTrue('L_F' in dict1) - self.assertEqual(dict1['L F'], dict1['L_F']) - self.assertTrue('a_b' in dict2) - self.assertEqual(dict2['a b'], dict2['a_b']) - self.assertTrue('L_M_H' in dict2) - self.assertEqual(dict2['L M H'], dict2['L_M_H']) - - def test_make_add_identifier(self): - """ - Test make_add_identifier for the .add methods py_name - """ - from pysd.translation.utils import make_add_identifier - - build_names = set() - - name = "values" - build_names.add(name) - - self.assertEqual(make_add_identifier(name, build_names), "valuesADD_1") - self.assertEqual(make_add_identifier(name, build_names), "valuesADD_2") - self.assertEqual(make_add_identifier(name, build_names), "valuesADD_3") - - name2 = "bb_a" - build_names.add(name2) - self.assertEqual(make_add_identifier(name2, build_names), "bb_aADD_1") - self.assertEqual(make_add_identifier(name, build_names), "valuesADD_4") - self.assertEqual(make_add_identifier(name2, build_names), "bb_aADD_2") - - def test_make_python_identifier(self): - from pysd.translation.utils import make_python_identifier - - self.assertEqual( - make_python_identifier('Capital'), 'capital') - - self.assertEqual( - make_python_identifier('multiple words'), 'multiple_words') - - self.assertEqual( - make_python_identifier('multiple spaces'), 'multiple_spaces') - - self.assertEqual( - make_python_identifier('for'), 'for_1') - - self.assertEqual( - make_python_identifier(' whitespace '), 'whitespace') - - self.assertEqual( - make_python_identifier('H@t tr!ck'), 'ht_trck') - - self.assertEqual( - make_python_identifier('123abc'), 'nvs_123abc') - - self.assertEqual( - make_python_identifier('Var$', {'Var$': 'var'}), - 'var') - - self.assertEqual( - make_python_identifier('Var@', {'Var$': 'var'}), 'var_1') - - self.assertEqual( - make_python_identifier('Var$', {'Var@': 'var', 'Var%': 'var_1'}), - 'var_2') - - my_vars = ["GDP 2010$", "GDP 2010€", "GDP 2010£"] - namespace = {} - expected = ["gdp_2010", "gdp_2010_1", "gdp_2010_2"] - for var, expect in zip(my_vars, expected): - self.assertEqual( - make_python_identifier(var, namespace), - expect) - - self.assertEqual( - make_python_identifier('1995 value'), - 'nvs_1995_value') - - self.assertEqual( - make_python_identifier('$ value'), - 'nvs_value') - - def test_make_coord_dict(self): - from pysd.translation.utils import make_coord_dict - self.assertEqual( - make_coord_dict(['Dim1', 'D'], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - terse=True), {'Dim2': ['D']}) - self.assertEqual( - make_coord_dict(['Dim1', 'D'], - {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D', 'E', 'F']}, - terse=False), {'Dim1': ['A', 'B', 'C'], - 'Dim2': ['D']}) - - def test_find_subscript_name(self): - from pysd.translation.utils import find_subscript_name - - self.assertEqual( - find_subscript_name({'Dim1': ['A', 'B'], - 'Dim2': ['C', 'D', 'E'], - 'Dim3': ['F', 'G', 'H', 'I']}, - 'D'), 'Dim2') - - self.assertEqual( - find_subscript_name({'Dim1': ['A', 'B'], - 'Dim2': ['C', 'D', 'E'], - 'Dim3': ['F', 'G', 'H', 'I']}, - 'Dim3'), 'Dim3') - - def test_make_merge_list(self): - from warnings import catch_warnings - from pysd.translation.utils import make_merge_list - - subscript_dict = { - "layers": ["l1", "l2", "l3"], - "layers1": ["l1", "l2", "l3"], - "up": ["l2", "l3"], - "down": ["l1", "l2"], - "dim": ["A", "B", "C"], - "dim1": ["A", "B", "C"] - } - - self.assertEqual( - make_merge_list([["l1"], ["up"]], - subscript_dict), - ["layers"]) - - self.assertEqual( - make_merge_list([["l3", "dim1"], ["down", "dim1"]], - subscript_dict), - ["layers", "dim1"]) - - self.assertEqual( - make_merge_list([["l2", "dim1", "dim"], ["l1", "dim1", "dim"]], - subscript_dict), - ["down", "dim1", "dim"]) - - self.assertEqual( - make_merge_list([["layers1", "l2"], ["layers1", "l3"]], - subscript_dict), - ["layers1", "up"]) - - # incomplete dimension - with catch_warnings(record=True) as ws: - self.assertEqual( - make_merge_list([["A"], ["B"]], - subscript_dict), - ["dim"]) - # use only user warnings - wu = [w for w in ws if issubclass(w.category, UserWarning)] - self.assertTrue(len(wu), 1) - self.assertIn("Dimension given by subscripts:" - + "\n\t{}\nis incomplete ".format({"A", "B"}) - + "using {} instead.".format("dim") - + "\nSubscript_dict:" - + "\n\t{}".format(subscript_dict), - str(wu[0].message)) - - # invalid dimension - try: - make_merge_list([["l1"], ["B"]], - subscript_dict) - self.assertFail() - except ValueError as err: - self.assertIn("Impossible to find the dimension that contains:" - + "\n\t{}\nFor subscript_dict:".format({"l1", "B"}) - + "\n\t{}".format(subscript_dict), - err.args[0]) - - # repeated subscript - with catch_warnings(record=True) as ws: - make_merge_list([["dim1", "A", "dim"], - ["dim1", "B", "dim"], - ["dim1", "C", "dim"]], - subscript_dict) - # use only user warnings - wu = [w for w in ws if issubclass(w.category, UserWarning)] - self.assertTrue(len(wu), 1) - self.assertIn( - "Adding new subscript range to subscript_dict:\ndim2: A, B, C", - str(wu[0].message)) - - subscript_dict2 = { - "dim1": ["A", "B", "C", "D"], - "dim1n": ["A", "B"], - "dim1y": ["C", "D"], - "dim2": ["E", "F", "G", "H"], - "dim2n": ["E", "F"], - "dim2y": ["G", "H"] - } - - # merging two subranges - self.assertEqual( - make_merge_list([["dim1y"], - ["dim1n"]], - subscript_dict2), - ["dim1"]) - - # final subscript in list - self.assertEqual( - make_merge_list([["dim1", "dim2n"], - ["dim1n", "dim2y"], - ["dim1y", "dim2y"]], - subscript_dict2), - ["dim1", "dim2"]) - - def test_update_dependency(self): - from pysd.translation.utils import update_dependency - - deps_dict = {} - - update_dependency("var1", deps_dict) - self.assertEqual(deps_dict, {"var1": 1}) - - update_dependency("var1", deps_dict) - self.assertEqual(deps_dict, {"var1": 2}) - - update_dependency("var2", deps_dict) - self.assertEqual(deps_dict, {"var1": 2, "var2": 1}) - - for i in range(10): - update_dependency("var1", deps_dict) - self.assertEqual(deps_dict, {"var1": 12, "var2": 1}) diff --git a/tests/unit_test_utils.py b/tests/unit_test_utils.py index bb3f1be6..41e8b6f6 100644 --- a/tests/unit_test_utils.py +++ b/tests/unit_test_utils.py @@ -133,6 +133,22 @@ def test_make_flat_df(self): self.assertEqual(set(actual.columns), set(expected.columns)) assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + def test_make_flat_df_0dxarray(self): + import pysd + + df = pd.DataFrame(index=[1], columns=['elem1']) + df.at[1] = [xr.DataArray(5)] + + expected = pd.DataFrame(index=[1], data={'Elem1': 5.}) + + return_addresses = {'Elem1': ('elem1', {})} + + actual = pysd.utils.make_flat_df(df, return_addresses, flatten=True) + + # check all columns are in the DataFrame + self.assertEqual(set(actual.columns), set(expected.columns)) + assert_frames_close(actual, expected, rtol=1e-8, atol=1e-8) + def test_make_flat_df_nosubs(self): import pysd @@ -238,6 +254,44 @@ def test_make_flat_df_flatten(self): actual.loc[:, col].values, expected.loc[:, col].values) + def test_make_flat_df_flatten_transposed(self): + import pysd + + df = pd.DataFrame(index=[1], columns=['elem2']) + df.at[1] = [ + xr.DataArray( + [[1, 4, 7], [2, 5, 8], [3, 6, 9]], + {'Dim2': ['D', 'E', 'F'], 'Dim1': ['A', 'B', 'C']}, + ['Dim2', 'Dim1'] + ).transpose("Dim1", "Dim2") + ] + + expected = pd.DataFrame(index=[1], columns=[ + 'Elem2[A,D]', + 'Elem2[A,E]', + 'Elem2[A,F]', + 'Elem2[B,D]', + 'Elem2[B,E]', + 'Elem2[B,F]', + 'Elem2[C,D]', + 'Elem2[C,E]', + 'Elem2[C,F]']) + + expected.at[1] = [1, 2, 3, 4, 5, 6, 7, 8, 9] + + return_addresses = { + 'Elem2': ('elem2', {})} + + actual = pysd.utils.make_flat_df(df, return_addresses, flatten=True) + print(actual.columns) + # check all columns are in the DataFrame + self.assertEqual(set(actual.columns), set(expected.columns)) + # need to assert one by one as they are xarrays + for col in set(expected.columns): + self.assertEqual( + actual.loc[:, col].values, + expected.loc[:, col].values) + def test_make_flat_df_times(self): import pysd diff --git a/tests/unit_test_vensim2py.py b/tests/unit_test_vensim2py.py deleted file mode 100644 index 498355f2..00000000 --- a/tests/unit_test_vensim2py.py +++ /dev/null @@ -1,1181 +0,0 @@ -import unittest -import xarray as xr - - -class TestGetFileSections(unittest.TestCase): - def test_normal_load(self): - """normal model file with no macros""" - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections(r"a~b~c| d~e~f| g~h~i|") - expected = [ - { - "returns": [], - "params": [], - "name": "_main_", - "string": "a~b~c| d~e~f| g~h~i|", - } - ] - self.assertEqual(actual, expected) - - def test_macro_only(self): - """ Macro Only """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections(":MACRO: MAC(z) a~b~c| :END OF MACRO:") - expected = [{"returns": [], "params": ["z"], "name": "MAC", - "string": "a~b~c|"}] - self.assertEqual(actual, expected) - - def test_macro_and_model(self): - """ basic macro and model """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections( - ":MACRO: MAC(z) a~b~c| :END OF MACRO: d~e~f| g~h~i|") - expected = [ - {"returns": [], "params": ["z"], "name": "MAC", - "string": "a~b~c|"}, - {"returns": [], "params": [], "name": "_main_", - "string": "d~e~f| g~h~i|"}, - ] - self.assertEqual(actual, expected) - - def test_macro_multiple_inputs(self): - """ macro with multiple input parameters """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections( - ":MACRO: MAC(z, y) a~b~c| :END OF MACRO: d~e~f| g~h~i|" - ) - expected = [ - {"returns": [], "params": ["z", "y"], "name": "MAC", - "string": "a~b~c|"}, - {"returns": [], "params": [], "name": "_main_", - "string": "d~e~f| g~h~i|"}, - ] - self.assertEqual(actual, expected) - - def test_macro_with_returns(self): - """ macro with return values """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections( - ":MACRO: MAC(z, y :x, w) a~b~c| :END OF MACRO: d~e~f| g~h~i|" - ) - expected = [ - { - "returns": ["x", "w"], - "params": ["z", "y"], - "name": "MAC", - "string": "a~b~c|", - }, - {"returns": [], "params": [], "name": "_main_", - "string": "d~e~f| g~h~i|"}, - ] - self.assertEqual(actual, expected) - - def test_handle_encoding(self): - """ Handle encoding """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections(r"{UTF-8} a~b~c| d~e~f| g~h~i|") - expected = [ - { - "returns": [], - "params": [], - "name": "_main_", - "string": "a~b~c| d~e~f| g~h~i|", - } - ] - self.assertEqual(actual, expected) - - def test_handle_encoding_like_strings(self): - """ Handle encoding-like strings in other places in the file """ - from pysd.translation.vensim.vensim2py import get_file_sections - - actual = get_file_sections(r"a~b~c| d~e~f{special}| g~h~i|") - expected = [ - { - "returns": [], - "params": [], - "name": "_main_", - "string": "a~b~c| d~e~f{special}| g~h~i|", - } - ] - self.assertEqual(actual, expected) - - -class TestEquationStringParsing(unittest.TestCase): - """ Tests the 'get_equation_components function """ - - def test_basics(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components(r'constant = 25'), - { - 'expr': '25', - 'kind': 'component', - 'subs': [], - 'subs_compatibility': {}, - 'real_name': 'constant', - 'keyword': None - } - ) - - def test_equals_handling(self): - """ Parse cases with equal signs within the expression """ - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components(r"Boolean = IF THEN ELSE(1 = 1, 1, 0)"), - { - "expr": "IF THEN ELSE(1 = 1, 1, 0)", - "kind": "component", - "subs": [], - "subs_compatibility": {}, - "real_name": "Boolean", - "keyword": None, - }, - ) - - def test_whitespace_handling(self): - """ Whitespaces should be shortened to a single space """ - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components( - r"""constant\t = - \t25\t """ - ), - { - "expr": "25", - "kind": "component", - "subs": [], - "subs_compatibility": {}, - "real_name": "constant", - "keyword": None, - }, - ) - - # test eliminating vensim's line continuation character - self.assertEqual( - get_equation_components( - r"""constant [Sub1, \\ - Sub2] = 10, 12; 14, 16;""" - ), - { - "expr": "10, 12; 14, 16;", - "kind": "component", - "subs": ["Sub1", "Sub2"], - "subs_compatibility": {}, - "real_name": "constant", - "keyword": None, - }, - ) - - def test_subscript_definition_parsing(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components(r"""Sub1: Entry 1, Entry 2, Entry 3 """), - { - "expr": None, - "kind": "subdef", - "subs": ["Entry 1", "Entry 2", "Entry 3"], - "subs_compatibility": {}, - "real_name": "Sub1", - "keyword": None, - }, - ) - - with self.assertRaises(ValueError) as err: - get_equation_components(r"""Sub2: (1-3) """) - - self.assertIn( - "A numeric range must contain at least one letter.", - str(err.exception)) - - with self.assertRaises(ValueError) as err: - get_equation_components(r"""Sub2: (a1-a1) """) - - self.assertIn( - "The number of the first subscript value must be " - "lower than the second subscript value in a " - "subscript numeric range.", - str(err.exception)) - - with self.assertRaises(ValueError) as err: - get_equation_components(r"""Sub2: (a1-b3) """) - - self.assertIn( - "Only matching names ending in numbers are valid.", - str(err.exception)) - - def test_subscript_references(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components( - r"constant [Sub1, Sub2] = 10, 12; 14, 16;"), - { - "expr": "10, 12; 14, 16;", - "kind": "component", - "subs": ["Sub1", "Sub2"], - "subs_compatibility": {}, - "real_name": "constant", - "keyword": None, - }, - ) - - self.assertEqual( - get_equation_components( - r"function [Sub1] = other function[Sub1]"), - { - "expr": "other function[Sub1]", - "kind": "component", - "subs": ["Sub1"], - "subs_compatibility": {}, - "real_name": "function", - "keyword": None, - }, - ) - - self.assertEqual( - get_equation_components( - r'constant ["S1,b", "S1,c"] = 1, 2; 3, 4;'), - { - "expr": "1, 2; 3, 4;", - "kind": "component", - "subs": ['"S1,b"', '"S1,c"'], - "subs_compatibility": {}, - "real_name": "constant", - "keyword": None, - }, - ) - - self.assertEqual( - get_equation_components( - r'constant ["S1=b", "S1=c"] = 1, 2; 3, 4;'), - { - "expr": "1, 2; 3, 4;", - "kind": "component", - "subs": ['"S1=b"', '"S1=c"'], - "subs_compatibility": {}, - "real_name": "constant", - "keyword": None, - }, - ) - - def test_lookup_definitions(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components(r"table([(0,-1)-(45,1)],(0,0),(5,0))"), - { - "expr": "([(0,-1)-(45,1)],(0,0),(5,0))", - "kind": "lookup", - "subs": [], - "subs_compatibility": {}, - "real_name": "table", - "keyword": None, - }, - ) - - self.assertEqual( - get_equation_components(r"table2 ([(0,-1)-(45,1)],(0,0),(5,0))"), - { - "expr": "([(0,-1)-(45,1)],(0,0),(5,0))", - "kind": "lookup", - "subs": [], - "subs_compatibility": {}, - "real_name": "table2", - "keyword": None, - }, - ) - - def test_get_lookup(self): - from pysd.translation.vensim.vensim2py import parse_lookup_expression - - res = parse_lookup_expression( - { - "expr": r"(GET DIRECT LOOKUPS('path2excel.xlsx', " - + r"'SheetName', 'index'\ , 'values'))", - "py_name": "get_lookup", - "subs": [], - "merge_subs": [] - }, - {} - )[1][0] - - self.assertEqual( - res["py_expr"], - "ExtLookup('path2excel.xlsx', 'SheetName', 'index', 'values', " - + "{},\n _root, '_ext_lookup_get_lookup')", - ) - - def test_pathological_names(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - self.assertEqual( - get_equation_components(r'"silly-string" = 25'), - { - "expr": "25", - "kind": "component", - "subs": [], - "subs_compatibility": {}, - "real_name": '"silly-string"', - "keyword": None, - }, - ) - - self.assertEqual( - get_equation_components(r'"pathological\\-string" = 25'), - { - "expr": "25", - "kind": "component", - "subs": [], - "subs_compatibility": {}, - "real_name": r'"pathological\\-string"', - "keyword": None, - }, - ) - - def test_get_equation_components_error(self): - from pysd.translation.vensim.vensim2py import get_equation_components - - defi = "NIF: NFNF" - try: - get_equation_components(defi) - self.assertFail() - except ValueError as err: - self.assertIn( - "\nError when parsing definition:\n\t %s\n\n" - "probably used definition is invalid or not integrated..." - "\nSee parsimonious output above." % defi, - err.args[0], - ) - - -class TestParse_general_expression(unittest.TestCase): - def test_arithmetic(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression({"expr": "-10^3+4"}) - self.assertEqual(res[0]["py_expr"], "-10**3+4") - - def test_arithmetic_scientific(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression({"expr": "1e+4"}) - self.assertEqual(res[0]["py_expr"], "1e+4") - - res = parse_general_expression({"expr": "2e4"}) - self.assertEqual(res[0]["py_expr"], "2e4") - - res = parse_general_expression({"expr": "3.43e04"}) - self.assertEqual(res[0]["py_expr"], "3.43e04") - - res = parse_general_expression({"expr": "1.0E4"}) - self.assertEqual(res[0]["py_expr"], "1.0E4") - - res = parse_general_expression({"expr": "-2.0E43"}) - self.assertEqual(res[0]["py_expr"], "-2.0E43") - - res = parse_general_expression({"expr": "-2.0e-43"}) - self.assertEqual(res[0]["py_expr"], "-2.0e-43") - - def test_caps_handling(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression({"expr": "Abs(-3)"}) - self.assertEqual(res[0]["py_expr"], "np.abs(-3)") - - res = parse_general_expression({"expr": "ABS(-3)"}) - self.assertEqual(res[0]["py_expr"], "np.abs(-3)") - - res = parse_general_expression({"expr": "aBS(-3)"}) - self.assertEqual(res[0]["py_expr"], "np.abs(-3)") - - def test_empty(self): - from warnings import catch_warnings - from pysd.translation.vensim.vensim2py import parse_general_expression - - with catch_warnings(record=True) as ws: - res = parse_general_expression({"expr": "", "real_name": "Var"}) - # use only user warnings - wu = [w for w in ws if issubclass(w.category, UserWarning)] - self.assertEqual(len(wu), 1) - self.assertIn("Empty expression for 'Var'", str(wu[0].message)) - - self.assertEqual(res[0]["py_expr"], "None") - - def test_function_calls(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression({"expr": "ABS(StockA)", - "real_name": "AB", - "eqn": "AB = ABS(StockA)"}, - {"StockA": "stocka"}) - self.assertEqual(res[0]["py_expr"], "np.abs(stocka())") - - res = parse_general_expression( - {"expr": "If Then Else(A>B, 1, 0)", - "real_name": "IFE", - "eqn": "IFE = If Then Else(A>B, 1, 0)"}, - {"A": "a", "B": "b"} - ) - self.assertEqual( - res[0]["py_expr"], "if_then_else(a()>b(), lambda: 1, lambda: 0)" - ) - - # test that function calls are handled properly in arguments - res = parse_general_expression( - {"expr": "If Then Else(A>B,1,A)"}, {"A": "a", "B": "b"} - ) - self.assertEqual( - res[0]["py_expr"], "if_then_else(a()>b(), lambda: 1, lambda: a())" - ) - - def test_id_parsing(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression({"expr": "StockA"}, - {"StockA": "stocka"}) - self.assertEqual(res[0]["py_expr"], "stocka()") - - def test_logicals(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :AND: 0,0,1)'}) - self.assertEqual(res[0]['py_expr'], - 'if_then_else(logical_and(1,0), lambda: 0, lambda: 1)' - ) - - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :OR: 0,0,1)'}) - self.assertEqual( - res[0]['py_expr'], - 'if_then_else(logical_or(1,0), lambda: 0, lambda: 1)' - ) - - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :AND: 0 :and: 1,0,1)'}) - self.assertEqual( - res[0]['py_expr'], - 'if_then_else(logical_and(1,0,1), lambda: 0, lambda: 1)' - ) - - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :or: 0 :OR: 1 :oR: 0,0,1)'}) - self.assertEqual( - res[0]['py_expr'], - 'if_then_else(logical_or(1,0,1,0), lambda: 0, lambda: 1)' - ) - - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :AND: (0 :OR: 1),0,1)'}) - self.assertEqual(res[0]['py_expr'], - 'if_then_else(logical_and(1,logical_or(0,1)),' + - ' lambda: 0, lambda: 1)') - - res = parse_general_expression( - {'expr': 'IF THEN ELSE((1 :AND: 0) :OR: 1,0,1)'}) - self.assertEqual(res[0]['py_expr'], - 'if_then_else(logical_or(logical_and(1,0),1),' + - ' lambda: 0, lambda: 1)') - - with self.assertRaises(ValueError): - res = parse_general_expression( - {'expr': 'IF THEN ELSE(1 :AND: 0 :OR: 1,0,1)', - 'real_name': 'logical', - 'eqn': 'logical = IF THEN ELSE(1 :AND: 0 :OR: 1,0,1)'}) - - def test_number_parsing(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - res = parse_general_expression({'expr': '20'}) - self.assertEqual(res[0]['py_expr'], '20') - - res = parse_general_expression({"expr": "3.14159"}) - self.assertEqual(res[0]["py_expr"], "3.14159") - - res = parse_general_expression({"expr": "1.3e+10"}) - self.assertEqual(res[0]["py_expr"], "1.3e+10") - - res = parse_general_expression({"expr": "-1.3e-10"}) - self.assertEqual(res[0]["py_expr"], "-1.3e-10") - - def test_nan_parsing(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - from pysd.translation.builder import Imports - - Imports.reset() - self.assertFalse(Imports._numpy) - res = parse_general_expression({'expr': ':NA:'}) - self.assertEqual(res[0]['py_expr'], 'np.nan') - self.assertTrue(Imports._numpy) - - def test_stock_construction_function_no_subscripts(self): - """ stock construction should create a stateful variable and - reference it """ - from pysd.translation.vensim.vensim2py import parse_general_expression - from pysd.py_backend.statefuls import Integ - - res = parse_general_expression( - { - "expr": "INTEG (FlowA, -10)", - "py_name": "test_stock", - "subs": [], - "merge_subs": [] - }, - {"FlowA": "flowa"} - ) - - self.assertEqual(res[1][0]["kind"], "stateful") - a = eval(res[1][0]["py_expr"]) - self.assertIsInstance(a, Integ) - - # check the reference to that variable - self.assertEqual(res[0]["py_expr"], res[1][0]["py_name"] + "()") - - def test_delay_construction_function_no_subscripts(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - from pysd.py_backend.statefuls import Delay - - res = parse_general_expression( - { - "expr": "DELAY1(Variable, DelayTime)", - "py_name": "test_delay", - "subs": [], - "merge_subs": [] - }, - { - "Variable": "variable", - "DelayTime": "delaytime", - "TIME STEP": "time_step", - } - ) - - def time_step(): - return 0.5 - - self.assertEqual(res[1][0]["kind"], "stateful") - a = eval(res[1][0]["py_expr"]) - self.assertIsInstance(a, Delay) - - # check the reference to that variable - self.assertEqual(res[0]["py_expr"], res[1][0]["py_name"] + "()") - - def test_forecast_construction_function_no_subscripts(self): - """ Tests translation of 'forecast' - - This translation should create a new stateful object to hold the - forecast elements, and then pass back a reference to that value - """ - from pysd.translation.vensim.vensim2py import parse_general_expression - from pysd.py_backend.statefuls import Forecast - - res = parse_general_expression( - { - "expr": "FORECAST(Variable, AverageTime, Horizon)", - "py_name": "test_forecast", - "subs": [], - "merge_subs": [] - }, - {"Variable": "variable", "AverageTime": "averagetime", - "Horizon": "horizon"}, - elements_subs_dict={"test_forecast": []}, - ) - - # check stateful object creation - self.assertEqual(res[1][0]["kind"], "stateful") - a = eval(res[1][0]["py_expr"]) - self.assertIsInstance(a, Forecast) - - # check the reference to that variable - self.assertEqual(res[0]["py_expr"], res[1][0]["py_name"] + "()") - - def test_smooth_construction_function_no_subscripts(self): - """ Tests translation of 'smooth' - - This translation should create a new stateful object to hold the delay - elements, and then pass back a reference to that value - """ - from pysd.translation.vensim.vensim2py import parse_general_expression - from pysd.py_backend.statefuls import Smooth - - res = parse_general_expression( - { - "expr": "SMOOTH(Variable, DelayTime)", - "py_name": "test_smooth", - "subs": [], - "merge_subs": [] - }, - {"Variable": "variable", "DelayTime": "delaytime"}, - ) - - # check stateful object creation - self.assertEqual(res[1][0]["kind"], "stateful") - a = eval(res[1][0]["py_expr"]) - self.assertIsInstance(a, Smooth) - - # check the reference to that variable - self.assertEqual(res[0]["py_expr"], res[1][0]["py_name"] + "()") - - def test_subscript_float_initialization(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - _subscript_dict = { - "Dim": ["A", "B", "C", "D", "E"], - "Dim1": ["A", "B", "C"], "Dim2": ["D", "E"] - } - - # case 1 - element = parse_general_expression( - {"expr": "3.32", "subs": ["Dim1"], "py_name": "var", - "merge_subs": ["Dim1"]}, {}, - _subscript_dict - - ) - string = element[0]["py_expr"] - # TODO we should use a = eval(string) - # hoewever eval is not detecting _subscript_dict variable - self.assertEqual( - string, - "xr.DataArray(3.32,{'Dim1': _subscript_dict['Dim1']},['Dim1'])", - ) - a = xr.DataArray( - 3.32, {dim: _subscript_dict[dim] for dim in ["Dim1"]}, ["Dim1"] - ) - self.assertDictEqual( - {key: list(val.values) for key, val in a.coords.items()}, - {"Dim1": ["A", "B", "C"]}, - ) - self.assertEqual(a.loc[{"Dim1": "B"}], 3.32) - - # case 2: xarray subscript is a subrange from the final subscript range - element = parse_general_expression( - {"expr": "3.32", "subs": ["Dim1"], "py_name": "var", - "merge_subs": ["Dim"]}, {}, _subscript_dict - ) - string = element[0]["py_expr"] - # TODO we should use a = eval(string) - # hoewever eval is not detecting _subscript_dict variable - self.assertEqual( - string, - "xr.DataArray(3.32,{'Dim': _subscript_dict['Dim1']},['Dim'])", - ) - a = xr.DataArray( - 3.32, {"Dim": _subscript_dict["Dim1"]}, ["Dim"] - ) - self.assertDictEqual( - {key: list(val.values) for key, val in a.coords.items()}, - {"Dim": ["A", "B", "C"]}, - ) - self.assertEqual(a.loc[{"Dim": "B"}], 3.32) - - def test_subscript_1d_constant(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - _subscript_dict = {"Dim1": ["A", "B", "C"], "Dim2": ["D", "E"]} - element = parse_general_expression( - {"expr": "1, 2, 3", "subs": ["Dim1"], "py_name": "var", - "merge_subs": ["Dim1"]}, - {}, _subscript_dict - ) - string = element[0]["py_expr"] - # TODO we should use a = eval(string) - # hoewever eval is not detecting _subscript_dict variable - self.assertEqual( - string, - "xr.DataArray([1.,2.,3.],{'Dim1': _subscript_dict['Dim1']}," - "['Dim1'])", - ) - a = xr.DataArray([1.0, 2.0, 3.0], - {dim: _subscript_dict[dim] for dim in ["Dim1"]}, - ["Dim1"]) - self.assertDictEqual( - {key: list(val.values) for key, val in a.coords.items()}, - {"Dim1": ["A", "B", "C"]}, - ) - self.assertEqual(a.loc[{"Dim1": "A"}], 1) - - def test_subscript_2d_constant(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - _subscript_dict = {"Dim1": ["A", "B", "C"], "Dim2": ["D", "E"]} - element = parse_general_expression( - {"expr": "1, 2; 3, 4; 5, 6;", "subs": ["Dim1", "Dim2"], - "merge_subs": ["Dim1", "Dim2"], "py_name": "var"}, - {}, _subscript_dict - ) - string = element[0]["py_expr"] - a = eval(string) - self.assertDictEqual( - {key: list(val.values) for key, val in a.coords.items()}, - {"Dim1": ["A", "B", "C"], "Dim2": ["D", "E"]}, - ) - self.assertEqual(a.loc[{"Dim1": "A", "Dim2": "D"}], 1) - self.assertEqual(a.loc[{"Dim1": "B", "Dim2": "E"}], 4) - - def test_subscript_3d_depth(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - _subscript_dict = {"Dim1": ["A", "B", "C"], "Dim2": ["D", "E"]} - element = parse_general_expression( - {"expr": "1, 2; 3, 4; 5, 6;", "subs": ["Dim1", "Dim2"], - "merge_subs": ["Dim1", "Dim2"], "py_name": "var"}, - {}, _subscript_dict, - ) - string = element[0]["py_expr"] - a = eval(string) - self.assertDictEqual( - {key: list(val.values) for key, val in a.coords.items()}, - {"Dim1": ["A", "B", "C"], "Dim2": ["D", "E"]}, - ) - self.assertEqual(a.loc[{"Dim1": "A", "Dim2": "D"}], 1) - self.assertEqual(a.loc[{"Dim1": "B", "Dim2": "E"}], 4) - - def test_subscript_builder(self): - """ - Testing how subscripts are translated when we have common subscript - ranges. - """ - from pysd.translation.vensim.vensim2py import\ - parse_general_expression, parse_lookup_expression - - _subscript_dict = { - "Dim1": ["A", "B", "C"], "Dim2": ["B", "C"], "Dim3": ["B", "C"] - } - - # case 1: subscript of the expr is in the final range, which is a - # subrange of a greater range - element = parse_general_expression( - {"py_name": "var1", "subs": ["B"], "real_name": "var1", "eqn": "", - "expr": "GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'C20')", - "merge_subs": ["Dim2"]}, - {}, - _subscript_dict - ) - self.assertIn( - "'Dim2': ['B']", element[1][0]['py_expr']) - - # case 1b: subscript of the expr is in the final range, which is a - # subrange of a greater range - element = parse_lookup_expression( - {"py_name": "var1b", "subs": ["B"], - "real_name": "var1b", "eqn": "", - "expr": "(GET DIRECT LOOKUPS('input.xlsx', 'Sheet1'," - " '19', 'C20'))", - "merge_subs": ["Dim2"]}, - _subscript_dict, - ) - self.assertIn( - "'Dim2': ['B']", element[1][0]['py_expr']) - - # case 2: subscript of the expr is a subscript subrange equal to the - # final range, which is a subrange of a greater range - element = parse_general_expression( - {"py_name": "var2", "subs": ["Dim2"], - "real_name": "var2", "eqn": "", - "expr": "GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'C20')", - "merge_subs": ["Dim2"]}, - {}, - _subscript_dict - ) - self.assertIn( - "'Dim2': _subscript_dict['Dim2']", element[1][0]['py_expr']) - - # case 3: subscript of the expr is a subscript subrange equal to the - # final range, which is a subrange of a greater range, but there is - # a similar subrange before - element = parse_general_expression( - {"py_name": "var3", "subs": ["B"], "real_name": "var3", "eqn": "", - "expr": "GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'C20')", - "merge_subs": ["Dim3"]}, - {}, - _subscript_dict - ) - self.assertIn( - "'Dim3': ['B']", element[1][0]['py_expr']) - - # case 4: subscript of the expr is a subscript subrange and the final - # subscript is a greater range - element = parse_general_expression( - {"py_name": "var4", "subs": ["Dim2"], - "real_name": "var4", "eqn": "", - "expr": "GET DIRECT CONSTANTS('input.xlsx', 'Sheet1', 'C20')", - "merge_subs": ["Dim1"]}, - {}, - _subscript_dict, - ) - self.assertIn( - "'Dim1': _subscript_dict['Dim2']", element[1][0]['py_expr']) - - # case 4b: subscript of the expr is a subscript subrange and the final - # subscript is a greater range - element = parse_general_expression( - {"py_name": "var4b", "subs": ["Dim2"], - "real_name": "var4b", "eqn": "", - "expr": "GET DIRECT DATA('input.xlsx', 'Sheet1', '19', 'C20')", - "keyword": None, "merge_subs": ["Dim1"]}, - {}, - _subscript_dict - ) - self.assertIn( - "'Dim1': _subscript_dict['Dim2']", element[1][0]['py_expr']) - - # case 4c: subscript of the expr is a subscript subrange and the final - # subscript is a greater range - element = parse_general_expression( - {"py_name": "var4c", "subs": ["Dim2"], - "real_name": "var4c", "eqn": "", - "expr": "GET DIRECT LOOKUPS('input.xlsx', 'Sheet1'," - " '19', 'C20')", "merge_subs": ["Dim1"]}, - {}, - _subscript_dict - ) - self.assertIn( - "'Dim1': _subscript_dict['Dim2']", element[1][0]['py_expr']) - - # case 4d: subscript of the expr is a subscript subrange and the final - # subscript is a greater range - element = parse_lookup_expression( - {"py_name": "var4d", "subs": ["Dim2"], - "real_name": "var4d", "eqn": "", - "expr": "(GET DIRECT LOOKUPS('input.xlsx', 'Sheet1'," - " '19', 'C20'))", "merge_subs": ["Dim1"]}, - _subscript_dict - ) - self.assertIn( - "'Dim1': _subscript_dict['Dim2']", element[1][0]['py_expr']) - - def test_subscript_reference(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - {"expr": "Var A[Dim1, Dim2]", "real_name": "Var2", "eqn": ""}, - {"Var A": "var_a"}, - {"Dim1": ["A", "B"], "Dim2": ["C", "D", "E"]}, - None, - {"var_a": ["Dim1", "Dim2"]} - ) - - self.assertEqual(res[0]["py_expr"], "var_a()") - - res = parse_general_expression( - {"expr": "Var B[Dim1, C]"}, - {"Var B": "var_b"}, - {"Dim1": ["A", "B"], "Dim2": ["C", "D", "E"]}, - None, - {"var_b": ["Dim1", "Dim2"]}, - ) - - self.assertEqual( - res[0]["py_expr"], - "rearrange(var_b().loc[:, 'C'].reset_coords(drop=True)," - "['Dim1'],_subscript_dict)", - ) - - res = parse_general_expression({'expr': 'Var B[A, C]'}, - {'Var B': 'var_b'}, - {'Dim1': ['A', 'B'], - 'Dim2': ['C', 'D', 'E']}, - None, - {'var_b': ['Dim1', 'Dim2']}) - - self.assertEqual( - res[0]['py_expr'], - "float(var_b().loc['A', 'C'])") - - res = parse_general_expression({'expr': 'Var C[Dim1, C, H]'}, - {'Var C': 'var_c'}, - {'Dim1': ['A', 'B'], - 'Dim2': ['C', 'D', 'E'], - 'Dim3': ['F', 'G', 'H', 'I']}, - None, - {'var_c': ['Dim1', 'Dim2', 'Dim3']}) - self.assertEqual( - res[0]["py_expr"], - "rearrange(var_c().loc[:, 'C', 'H'].reset_coords(drop=True)," - "['Dim1'],_subscript_dict)", - ) - - res = parse_general_expression({'expr': 'Var C[B, C, H]'}, - {'Var C': 'var_c'}, - {'Dim1': ['A', 'B'], - 'Dim2': ['C', 'D', 'E'], - 'Dim3': ['F', 'G', 'H', 'I']}, - None, - {'var_c': ['Dim1', 'Dim2', 'Dim3']}) - - self.assertEqual( - res[0]['py_expr'], - "float(var_c().loc['B', 'C', 'H'])") - - def test_subscript_ranges(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - {"expr": "Var D[Range1]"}, - {"Var D": "var_c"}, - {"Dim1": ["A", "B", "C", "D", "E", "F"], - "Range1": ["C", "D", "E"]}, - None, - {"var_c": ["Dim1"]}, - ) - - self.assertEqual( - res[0]["py_expr"], "rearrange(var_c(),['Range1'],_subscript_dict)" - ) - - def test_invert_matrix(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - { - "expr": "INVERT MATRIX(A, 3)", - "real_name": "A1", - "py_name": "a1", - "merge_subs": ["dim1", "dim2"] - }, - { - "A": "a", - "A1": "a1", - }, - subscript_dict={ - "dim1": ["a", "b", "c"], "dim2": ["a", "b", "c"] - } - ) - - self.assertEqual(res[0]["py_expr"], "invert_matrix(a())") - - def test_subscript_elmcount(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - { - "expr": "ELMCOUNT(dim1)", - "real_name": "A", - "py_name": "a", - "merge_subs": [] - }, - { - "A": "a", - }, - subscript_dict={ - "dim1": ["a", "b", "c"], "dim2": ["a", "b", "c"] - } - ) - - self.assertIn( - "len(_subscript_dict['dim1'])", - res[0]["py_expr"], ) - - def test_subscript_logicals(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - res = parse_general_expression( - { - "expr": "IF THEN ELSE(dim1=dim2, 5, 0)", - "real_name": "A", - "py_name": "a", - "merge_subs": ["dim1", "dim2"] - }, - { - "A": "a", - }, - subscript_dict={ - "dim1": ["a", "b", "c"], "dim2": ["a", "b", "c"] - } - ) - - self.assertIn( - "xr.DataArray(_subscript_dict['dim1']," - "{'dim1': _subscript_dict['dim1']},'dim1')" - "==xr.DataArray(_subscript_dict['dim2']," - "{'dim2': _subscript_dict['dim2']},'dim2')", - res[0]["py_expr"], ) - - def test_ref_with_subscript_prefix(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - # When parsing functions arguments first the subscript ranges are - # parsed and later the general id is used, however, the if a reference - # to a var starts with a subscript range name this could make the - # parser crash - res = parse_general_expression( - { - "expr": "ABS(Upper var)", - "real_name": "A", - "eqn": "A = ABS(Upper var)", - "py_name": "a", - "merge_subs": [] - }, - { - "Upper var": "upper_var", - }, - subscript_dict={ - "upper": ["a", "b", "c"] - } - ) - - self.assertIn( - "np.abs(upper_var())", - res[0]["py_expr"], ) - - def test_random_0_1(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - # When parsing functions arguments first the subscript ranges are - # parsed and later the general id is used, however, the if a reference - # to a var starts with a subscript range name this could make the - # parser crash - res = parse_general_expression( - { - "expr": "RANDOM 0 1()", - "real_name": "A", - "eqn": "A = RANDOM 0 1()", - "py_name": "a", - "merge_subs": [], - "dependencies": set() - }, - { - "A": "a", - } - ) - - self.assertIn( - "np.random.uniform(0, 1)", - res[0]["py_expr"], ) - - def test_random_uniform(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - # When parsing functions arguments first the subscript ranges are - # parsed and later the general id is used, however, the if a reference - # to a var starts with a subscript range name this could make the - # parser crash - res = parse_general_expression( - { - "expr": "RANDOM UNIFORM(10, 15, 3)", - "real_name": "A", - "eqn": "A = RANDOM UNIFORM(10, 15, 3)", - "py_name": "a", - "merge_subs": [], - "dependencies": set() - }, - { - "A": "a", - } - ) - - self.assertIn( - "np.random.uniform(10, 15)", - res[0]["py_expr"], ) - - def test_incomplete_expression(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - from warnings import catch_warnings - - with catch_warnings(record=True) as w: - res = parse_general_expression( - { - "expr": "A FUNCTION OF(Unspecified Eqn,Var A,Var B)", - "real_name": "Incomplete Func", - "py_name": "incomplete_func", - "eqn": "Incomplete Func = A FUNCTION OF(Unspecified " - + "Eqn,Var A,Var B)", - "subs": [], - "merge_subs": [] - }, - { - "Unspecified Eqn": "unspecified_eqn", - "Var A": "var_a", - "Var B": "var_b", - } - ) - self.assertEqual(len(w), 1) - self.assertTrue( - "Incomplete Func has no equation specified" in - str(w[-1].message) - ) - - self.assertEqual(res[0]["py_expr"], - "incomplete(unspecified_eqn(), var_a(), var_b())") - - def test_parse_general_expression_error(self): - from pysd.translation.vensim.vensim2py import parse_general_expression - - element = { - "expr": "NIF(1,3)", - "real_name": "not implemented function", - "eqn": "not implemented function=\tNIF(1,3)", - } - try: - parse_general_expression(element) - self.assertFail() - except ValueError as err: - self.assertIn( - "\nError when parsing %s with equation\n\t %s\n\n" - "probably a used function is not integrated..." - "\nSee parsimonious output above." - % (element["real_name"], element["eqn"]), - err.args[0], - ) - - -class TestParse_sketch_line(unittest.TestCase): - def test_parse_sketch_line(self): - from pysd.translation.vensim.vensim2py import parse_sketch_line - - namespace = {'"var-n"': "varn", "Stock": "stock", '"rate-1"': "rate1"} - lines = [ - '10,1,"var-n",332,344,21,12,0,3,0,32,1,0,0,0,-1--1--1,0-0-0' + - ',@Malgun Gothic|12||0-0-0', # normal variable with colors - "10,2,Stock,497,237,40,20,3,3,0,0,0,0,0,0", # stock - '10,7,"rate-1",382,262,21,11,40,3,0,0,-1,0,0,0', # normal variable - '10,2,"var-n",235,332,27,11,8,2,0,3,-1,0,0,0,128-128-128,0-0-0,' + - '|0||128-128-128', # shadow variable - "*Just another view", # module definition - "1,5,6,3,100,0,0,22,0,0,0,-1--1--1,,1|(341,243)|", # arrow - "This is a random comment." - ] - - expected_var = [ - namespace['"var-n"'], - namespace["Stock"], - namespace['"rate-1"'], - "", - "", - "", - "" - ] - expected_mod = ["", "", "", "", "Just another view", "", ""] - - for num, line in enumerate(lines): - res = parse_sketch_line(line.strip(), namespace) - self.assertEqual(res["variable_name"], expected_var[num]) - self.assertEqual(res["view_name"], expected_mod[num]) - - -class TestParse_private_functions(unittest.TestCase): - def test__split_sketch_warning(self): - import warnings - from pysd.translation.vensim.vensim2py import _split_sketch - - model_str = "this is my model" - - with warnings.catch_warnings(record=True) as ws: - text, sketch = _split_sketch(model_str) - - # use only user warnings - wu = [w for w in ws if issubclass(w.category, UserWarning)] - self.assertEqual(len(wu), 1) - self.assertTrue( - "Your model does not have a sketch." in str(wu[0].message)) - - self.assertEqual(text, model_str) - self.assertEqual(sketch, "") diff --git a/tests/unit_test_xmile2py.py b/tests/unit_test_xmile2py.py deleted file mode 100644 index 7cdc7379..00000000 --- a/tests/unit_test_xmile2py.py +++ /dev/null @@ -1,67 +0,0 @@ -import os -import unittest -import tempfile - -from pysd.translation.xmile.xmile2py import translate_xmile - -_root = os.path.dirname(__file__) -TARGET_STMX_FILE = os.path.join(_root, "test-models/tests/game/test_game.stmx") - - -class TestXmileConversion(unittest.TestCase): - - def test_python_file_creation(self): - with open(TARGET_STMX_FILE, "r") as stmx: - contents = stmx.read() - - # Write out contents to temporary file - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_file.write(contents) - - # Convert file (should not raise error) - generated_file = translate_xmile(temp_file.name) - - # Check if both source file and python file exists - try: - assert generated_file != temp_file.name,\ - "Accidental replacement of original model file!" - assert generated_file.endswith(".py"),\ - "File created without python extension" - assert os.path.exists(temp_file.name)\ - and os.path.exists(generated_file),\ - "Expected files are missing" - finally: - os.remove(temp_file.name) - - try: - os.remove(generated_file) - except FileNotFoundError: - # Okay if python file is missing - pass - - def test_multiline_equation(self): - with open(TARGET_STMX_FILE, "r") as stmx: - contents = stmx.read() - - # Insert line break in equation definition - contents = contents.replace( - "(Stock+Constant)", - "(Stock+\nConstant)") - - # Write out contents to temporary file - with tempfile.NamedTemporaryFile(mode="w", delete=False) as temp_file: - temp_file.write(contents) - - # Convert file (should not raise error) - generated_file = translate_xmile(temp_file.name) - - with open(generated_file, "r") as fp: - contents = fp.read() - - idx = contents.find("stock() + constant()") - - try: - assert idx > 0, "Correct, generated, equation not found" - finally: - os.remove(temp_file.name) - os.remove(generated_file)