fix: Using custom filesystem produce error even though the template i…

…s render fine. E.g. when include in-memmory-common-head.html: …/templates/in-memmory-common-head.html(1,0) Error: Cannot stat file ‘…/templates/in-memmory-common-head.html' (No such file or directory) (gdb) bt #0 Teng::FileStat_t::stat (this=0x7fffffe85510, pos=..., err=...) at tengsourcelist.cc:56 seznam#1 0x00007ffff7b237ce in Teng::SourceList_t::addSource (this=0x637178, _source=<error reading variable: Cannot access memory at address 0xffffffffffffffe8>, pos=..., err=...) at tengsourcelist.cc:101 seznam#2 0x00007ffff7b6245a in Teng::Program_t::addSource (pos=..., source="/Volumes/UserData/workspace/github.com/teng-example/build/templates/in-memmory-common-head.html", this=<optimized out>) at tengprogram.h:86 seznam#3 Teng::tengSyntax_parse (context=context@entry=0x7fffffffdfe0) at tengsyntax.yy:551 seznam#4 0x00007ffff7b30743 in Teng::ParserContext_t::createProgramFromString (this=0x7fffffffdfe0, str=...) at tengparsercontext.cc:192 seznam#5 0x00007ffff7b5ca63 in Teng::TemplateCache_t::createTemplate (this=0x638ba8, templateSource=<error reading variable: Cannot access memory at address 0xffffffffffffffe8>, langFilename="", configFilename="\320\236\335\367\377\177\000\000hcc", '\000' <repeats 38 times>, sourceType=Teng::TemplateCache_t::SRC_FILE, sourceType@entry=Teng::TemplateCache_t::SRC_STRING) at tengtemplate.cc:115 seznam#6 0x00007ffff7b1c602 in Teng::Teng_t::generatePage (this=0x7fffffffe960, templateString="\n<html>\n<?teng include file=\"in-memmory-common-head.html\" ?>\n <body>\n <?teng frag row?><p>${rnum}\n", ' ' <repeats 12 times>, "<?teng frag col?>${cnum} <?teng endfrag?>\n </p><?teng endfrag?>\n </bod"..., dict=..., lang="", param="", scontentType=..., encoding="utf-8", data=..., writer=..., err=...) at teng.cc:224 seznam#7 0x0000000000402d34 in generate (teng=..., templ="\n<html>\n<?teng include file=\"in-memmory-common-head.html\" ?>\n <body>\n <?teng frag row?><p>${rnum}\n", ' ' <repeats 12 times>, "<?teng frag col?>${cnum} <?teng endfrag?>\n </p><?teng endfrag?>\n </bod"..., root=...) at /Volumes/UserData/workspace/github.com/teng-example/cpp/teng_utils.h:46 seznam#8 0x0000000000402153 in main (argc=1, argv=0x7fffffffeac8) at /Volumes/UserData/workspace/github.com/teng-example/cpp/teng_filesystem.cc:47
fboranek · Sep 14, 2018 · 0cabcd5 · 0cabcd5
1 parent 46763dc
commit 0cabcd5
Show file tree

Hide file tree

Showing 11 changed files with 125 additions and 71 deletions.
diff --git a/python/teng.cc b/python/teng.cc
@@ -107,16 +107,35 @@ class FilesystemWrapper_t : public FilesystemInterface_t {
     }
 
     ~FilesystemWrapper_t() { Py_XDECREF(callback); }
+
+    static bool checkObject(PyObject* callback)
+    {
+        PyObject* func_name = PyString_FromString("read");
+        if (!PyObject_HasAttr(callback, func_name)) {
+            Py_XINCREF(func_name);
+            PyErr_SetString(PyExc_TypeError, "parameter fileSystem must have function read(name)");
+            return false;
+        }
+        Py_XINCREF(func_name);
+
+        func_name = PyString_FromString("hash");
+        if (!PyObject_HasAttr(callback, func_name)) {
+            Py_XINCREF(func_name);
+            PyErr_SetString(PyExc_TypeError, "parameter fileSystem must have function hash(name)");
+            return false;
+        }
+        Py_XINCREF(func_name);
+
+        return true;
+    }
 
     virtual std::string read(const std::string& filename) const
     {
         std::string result;
 
-        PyObject* arglist = Py_BuildValue("(z)", filename.c_str());
-        PyObject* str = PyObject_CallObject(callback, arglist);
-        Py_DECREF(arglist);
+        PyObject* str = PyObject_CallMethod(callback, "read", "z", filename.c_str());
         if (!str) {
-            throw std::runtime_error("Failed to call python fileSystem callback.");
+            throw std::runtime_error("Failed to call python fileSystem.read callback.");
         }
 
         int res = getString(result, str);
@@ -133,6 +152,32 @@ class FilesystemWrapper_t : public FilesystemInterface_t {
         Py_DECREF(str);
         return result;
     }
+
+    virtual size_t hash(const std::string& filename) const
+    {
+        size_t result = 0;
+
+        PyObject* data = PyObject_CallMethod(callback, "hash", "z", filename.c_str());
+        if (!data) {
+            throw std::runtime_error("Failed to call python fileSystem.hash callback.");
+        }
+
+        if (PyInt_Check(data)) {
+            result = PyInt_AsLong(data);
+        } else if (PyLong_Check(data)) {
+            result = PyLong_AsLongLong(data);
+        }
+        else {
+            Py_DECREF(data);
+            throw std::runtime_error("Result of fileSystem.hash callback is not integer.");
+        }
+        if (PyErr_Occurred()) {
+            Py_DECREF(data);
+            throw std::runtime_error("Failed to convert result of python fileSystem.hash callback.");
+        }
+        Py_DECREF(data);
+        return result;
+    }
 
     static int getString(std::string& value, PyObject* data)
     {
@@ -280,8 +325,7 @@ PyObject* Teng_Teng(TengObject *self, PyObject *args, PyObject *keywds) {
         return 0;
 
     if (fileSystem) {
-        if (!PyCallable_Check(fileSystem)) {
-            PyErr_SetString(PyExc_TypeError, "parameter fileSystem must be callable");
+        if (!FilesystemWrapper_t::checkObject(fileSystem)) {
             return 0;
         }
     }

diff --git a/src/tengdictionary.cc b/src/tengdictionary.cc
@@ -425,7 +425,7 @@ int Dictionary_t::parse(const FilesystemInterface_t *filesystem,
         filename = root + '/' + filename;
 
     // insert source into source list
-    sources.addSource(filename, pos, err);
+    sources.addSource(filesystem, filename, pos, err);
 
     try {
         Error_t::Position_t newPos(filename, 1);

diff --git a/src/tengdictionary.h b/src/tengdictionary.h
@@ -103,8 +103,8 @@ class Dictionary_t {
      *
      * @return 0 OK !0 changed
      */
-    inline int check() const {
-        return sources.isChanged();
+    inline int check(const FilesystemInterface_t* filesystem) const {
+        return sources.isChanged(filesystem);
     }
 
     /**

diff --git a/src/tengfilesystem.cc b/src/tengfilesystem.cc
@@ -17,22 +17,27 @@
  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
  */
 
-#include <stdio.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
 #include <stdexcept>
+#include <stdio.h>
 
 #include "tengfilesystem.h"
 #include "tengutil.h"
 
+#include <boost/functional/hash.hpp>
+
 namespace Teng {
 
-std::string Filesystem_t::read(const std::string &filename_) const
+std::string Filesystem_t::read(const std::string& filename_) const
 {
     std::string result;
 
-    std::string filename (filename_);
+    std::string filename(filename_);
     tengNormalizeFilename(filename);
 
-    FILE *fp = fopen(filename.c_str(), "rb");
+    FILE* fp = fopen(filename.c_str(), "rb");
     if (fp == 0) throw std::runtime_error("fopen(" + filename + ")");
 
     char buf[1024];
@@ -50,4 +55,27 @@ std::string Filesystem_t::read(const std::string &filename_) const
     return result;
 }
 
+size_t Filesystem_t::hash(const std::string& filename) const
+{
+    std::size_t seed = 0;
+
+    // stat given file
+    struct stat buf;
+    if (::stat(filename.c_str(), &buf)) {
+        throw std::runtime_error("Cannot stat file '" + filename + "'");
+    }
+
+    // check if not dir
+    if (S_ISDIR(buf.st_mode)) {
+        throw std::runtime_error("File '" + filename + "' is a directory");
+    }
+
+    boost::hash_combine(seed, buf.st_ino);
+    boost::hash_combine(seed, buf.st_size);
+    boost::hash_combine(seed, buf.st_mtime);
+    boost::hash_combine(seed, buf.st_ctime);
+
+    return seed;
+}
+
 } // namespace Teng
diff --git a/src/tengfilesystem.h b/src/tengfilesystem.h
@@ -35,6 +35,7 @@ class FilesystemInterface_t {
      * @return Contents of the file in filesystem
      */
     virtual std::string read(const std::string &filename) const = 0;
+    virtual size_t hash(const std::string &filename) const = 0;
 
     virtual ~FilesystemInterface_t() {}
 };
@@ -44,6 +45,7 @@ class FilesystemInterface_t {
 class Filesystem_t : public FilesystemInterface_t {
 public:
     virtual std::string read(const std::string &filename) const;
+    virtual size_t hash(const std::string &filename) const;
 };
 
 /** @short Implementation of filesystem interface backed by key-value storage.
@@ -54,6 +56,11 @@ class InMemoryFilesystem_t : public FilesystemInterface_t {
     {
         return storage.at(filename);
     }
+
+    virtual size_t hash(const std::string &filename) const
+    {
+        return 0; // permanent cache
+    }
 
     /** @short Key-value storage.
      */

diff --git a/src/tengparsercontext.cc b/src/tengparsercontext.cc
@@ -114,7 +114,7 @@ Program_t* ParserContext_t::createProgramFromFile(
         path = filename;
     // add source to source list -- main template will always have index 0
     // and also remember returned sourceindex
-    sourceIndex.push(program->addSource(path, Error_t::Position_t()));
+    sourceIndex.push(program->addSource(filesystem, path, Error_t::Position_t()));
 
     // create first level-1 lexical analyzer (from file)
     lex1.push(new Lex1_t(filesystem, path, Error_t::Position_t("", 0, 0),

diff --git a/src/tengprogram.h b/src/tengprogram.h
@@ -62,8 +62,8 @@ class Program_t : private std::vector<Instruction_t> {
 
     /** @short Check source files for change.
       * @return 0=OK !0=changed. */
-    inline int check() const {
-        return sources.isChanged();
+    inline int check(const FilesystemInterface_t* filesystem) const {
+        return sources.isChanged(filesystem);
     }
 
     /** @short Return error log.
@@ -81,9 +81,10 @@ class Program_t : private std::vector<Instruction_t> {
     /** @short Adds new source into the list.
       * @param source Filename of source.
       * @param pos Position in current file. */
-    inline unsigned int addSource(const std::string &source,
+    inline unsigned int addSource(const FilesystemInterface_t* filesystem,
+                           const std::string &source,
                            const Error_t::Position_t &pos) {
-        return sources.addSource(source, pos, error);
+        return sources.addSource(filesystem,source, pos, error);
     }
 
     /** Get source's filename based on index in source list.

diff --git a/src/tengsourcelist.cc b/src/tengsourcelist.cc
@@ -45,40 +45,29 @@
 
 namespace Teng {
 
-int FileStat_t::stat(const Error_t::Position_t &pos,
+int FileStat_t::stat(const FilesystemInterface_t* filesystem,
+                     const Error_t::Position_t &pos,
                      Error_t &err)
 {
     // invalidate data;
     valid = false;
-
-    // stat given file
-    struct stat buf;
-    if (::stat(filename.c_str(), &buf)) {
-        err.logSyscallError(Error_t::LL_ERROR, pos, "Cannot stat file '" +
-                            filename +"'");
-        return -1;
+
+    try {
+        hash = filesystem->hash(filename);
     }
-
-    // check if not dir
-    if (S_ISDIR(buf.st_mode)) {
-        err.logError(Error_t::LL_ERROR, pos, "File '" + filename +
-                     "' is a directory");
+    catch(std::exception& ex) {
+        err.logSyscallError(Error_t::LL_ERROR, pos, ex.what());
         return -1;
     }
-
-    // populate members of fileInfo from stat
-    inode = buf.st_ino;
-    size = buf.st_size;
-    mtime = buf.st_mtime;
-    ctime = buf.st_ctime;
-
+
     // validate data
     valid = true;
     // OK
     return 0;
 }
 
-unsigned int SourceList_t::addSource(const std::string &_source,
+unsigned int SourceList_t::addSource(const FilesystemInterface_t* filesystem,
+                                     const std::string &_source,
                                      const Error_t::Position_t &pos,
                                      Error_t &err)
 {
@@ -98,22 +87,22 @@ unsigned int SourceList_t::addSource(const std::string &_source,
     }
 
     // stat file
-    fs.stat(pos, err);
+    fs.stat(filesystem, pos, err);
 
     // push info into source list
     sources.push_back(fs);
     return sources.size() - 1;
 }
 
-bool SourceList_t::isChanged() const {
+bool SourceList_t::isChanged(const FilesystemInterface_t* filesystem) const {
     Error_t err;
     Error_t::Position_t pos;
     // run through source list
     for (std::vector<FileStat_t>::const_iterator isources = sources.begin();
          isources != sources.end(); ++isources) {
         // stat file
         FileStat_t fs(isources->filename);
-        if (fs.stat(pos, err) && isources->valid)
+        if (fs.stat(filesystem, pos, err) && isources->valid)
             return true;
         // compare with cached value
         if (fs != *isources) return true;

diff --git a/src/tengsourcelist.h b/src/tengsourcelist.h
@@ -44,6 +44,7 @@
 #include <sys/types.h>
 
 #include "tengerror.h"
+#include "tengfilesystem.h"
 
 namespace Teng {
 
@@ -59,8 +60,7 @@ struct FileStat_t {
      * @param filename associated file name.
      */
     FileStat_t(const std::string &filename = std::string())
-        : filename(filename), inode(0), size(0),
-          mtime(0), ctime(0), valid(false)
+        : filename(filename), hash(0), valid(false)
     {}
 
 
@@ -71,7 +71,8 @@ struct FileStat_t {
      * @param err error logger
      * @return 0 OK !0 error
      */
-    int stat(const Error_t::Position_t &pos,
+    int stat(const FilesystemInterface_t* filesystem,
+             const Error_t::Position_t &pos,
              Error_t &err);
 
     /**
@@ -81,9 +82,7 @@ struct FileStat_t {
      * @return true if values are the same false otherwise
      */
     bool operator==(const FileStat_t &fs) const {
-        return ((filename == fs.filename) && (inode == fs.inode) &&
-                (size == fs.size) && (mtime == fs.mtime) &&
-                (ctime == fs.ctime));
+        return ((filename == fs.filename) && (hash == fs.hash));
     }
 
     /**
@@ -112,24 +111,9 @@ struct FileStat_t {
     std::string filename;
 
     /**
-     * @short Inode of file.
+     * @short Hash of file.
      */
-    ino_t inode;
-
-    /**
-     * @short Size of file.
-     */
-    off_t size;
-
-    /**
-     * @short Last modification of file.
-     */
-    time_t mtime;
-
-    /**
-     * @short Last attribute modification of file.
-     */
-    time_t ctime;
+    size_t hash;
 
     /**
      * @short Indicates that data came from stat(2).
@@ -155,7 +139,8 @@ class SourceList_t {
      * @param err error logger
      * @return position of added source in list
      */
-    unsigned int addSource(const std::string &source,
+    unsigned int addSource(const FilesystemInterface_t* filesystem,
+                           const std::string &source,
                            const Error_t::Position_t &pos,
                            Error_t &err);
 
@@ -165,7 +150,7 @@ class SourceList_t {
      *
      * @return true means modified; false not modified or error
      */
-    bool isChanged() const;
+    bool isChanged(const FilesystemInterface_t* filesystem) const;
 
     /** @short Get source by given index.
      *

diff --git a/src/tengsyntax.yy b/src/tengsyntax.yy
@@ -547,7 +547,7 @@ teng_include:
                 CONTEXT->lex2 = 0; //for sure
                 // append source list
                 CONTEXT->sourceIndex.push( //remember source index
-                        CONTEXT->program->addSource(fname,
+                        CONTEXT->program->addSource(CONTEXT->filesystem, fname,
                         CONTEXT->lex1.top()->getPosition()));
             }
         }