Merge pull request #44 from mappu/miqt-scintilla

QScintilla (GPL) and ScintillaEdit (MIT) support

.github/workflows/miqt.yml

@@ -15,7 +15,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Linux64 docker build
-      run: cd docker && docker build -t miqt/genbindings:latest -f genbindings.Dockerfile .
+      run: docker build -t miqt/genbindings:latest -f docker/genbindings.Dockerfile .
 
     - name: Cache clang ASTs
       uses: actions/cache@v4
@@ -37,7 +37,7 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Linux64 docker build
-      run: cd docker && docker build -t miqt/linux64:latest -f linux64-go1.19-qt5.15-dynamic.Dockerfile .
+      run: docker build -t miqt/linux64:latest -f docker/linux64-go1.19-qt5.15-dynamic.Dockerfile .
 
     - name: Cache GOCACHE
       uses: actions/cache@v4
@@ -62,7 +62,7 @@ jobs:
         key: win64-gocache
 
     - name: Win64 docker build
-      run: cd docker && docker build -t miqt/win64:latest -f win64-cross-go1.23-qt5.15-static.Dockerfile .
+      run: docker build -t miqt/win64:latest -f docker/win64-cross-go1.23-qt5.15-static.Dockerfile .
 
     - name: Win64 bindings compile
       run: docker run -v ~/.cache/go-build:/root/.cache/go-build -v $PWD:/src -w /src miqt/win64:latest /bin/bash -c 'cd qt && go build'

.gitignore

@@ -7,6 +7,9 @@ container-build-cache/
 # local genbindings configuration
 cmd/genbindings/genbindings.local*
 
+# local pkg-config configuration
+pkg-config/*.pc
+
 # binaries
 cmd/handbindings/handbindings
 cmd/handbindings/bindings_test/direct
@@ -23,6 +26,8 @@ examples/windowsmanifest/windowsmanifest.exe
 examples/uidesigner/uidesigner
 examples/uidesigner/uidesigner.exe
 examples/libraries/qt-qprintsupport/qt-qprintsupport
+examples/libraries/restricted-extras-qscintilla/restricted-extras-qscintilla
+examples/libraries/extras-scintillaedit/extras-scintillaedit
 
 # android temporary build files
 android-build

README.md

@@ -149,14 +149,14 @@ Static builds are also available by installing the `mingw-w64-ucrt-x86_64-qt5-st
 For static builds (open source application):
 
 1. Build the necessary docker container for cross-compilation:
-	- `docker build -t miqt/win64-cross:latest -f win64-cross-go1.23-qt5.15-static.Dockerfile .`
+	- `docker build -t miqt/win64-cross:latest -f docker/win64-cross-go1.23-qt5.15-static.Dockerfile .`
 2. Build your application:
 	- `docker run --rm -v $(pwd):/src -w /src miqt/win64-cross:latest go build -buildvcs=false --tags=windowsqtstatic -ldflags '-s -w -H windowsgui'`
 
 For dynamically-linked builds (closed-source or open source application):
 
 1. Build the necessary docker container for cross-compilation:
-	- `docker build -t miqt/win64-dynamic:latest -f win64-cross-go1.23-qt5.15-dynamic.Dockerfile .`
+	- `docker build -t miqt/win64-dynamic:latest -f docker/win64-cross-go1.23-qt5.15-dynamic.Dockerfile .`
 2. Build your application:
 	- `docker run --rm -v $(pwd):/src -w /src miqt/win64-dynamic:latest go build -buildvcs=false -ldflags '-s -w -H windowsgui'`
 3. Copy necessary Qt LGPL libraries and plugin files.
@@ -179,7 +179,7 @@ Miqt supports compiling for Android. Some extra steps are required to bridge the
 	- Ensure to `import "C"`.
 	- Check `examples/android` to see how to support both Android and desktop platforms.
 2. Build the necessary docker container for cross-compilation:
-	- `docker build -t miqt/android:latest -f android-armv8a-go1.23-qt5.15-dynamic.Dockerfile .`
+	- `docker build -t miqt/android:latest -f docker/android-armv8a-go1.23-qt5.15-dynamic.Dockerfile .`
 3. Build your application as `.so` format:
 	- `docker run --rm -v $(pwd):/src -w /src miqt/android:latest go build -buildmode c-shared -ldflags "-s -w -extldflags -Wl,-soname,my_go_app.so" -o android-build/libs/arm64-v8a/my_go_app.so`
 4. Build the Qt linking stub:

cmd/genbindings/clangexec.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"strings"
 	"sync"
 	"time"
 )
@@ -18,7 +19,26 @@ const (
 	ClangRetryDelay = 3 * time.Second
 )
 
-func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []string) ([]interface{}, error) {
+type ClangMatcher func(astNodeFilename string) bool
+
+func ClangMatchSameHeaderDefinitionOnly(astNodeFilename string) bool {
+	return astNodeFilename == ""
+}
+
+type clangMatchUnderPath struct {
+	basePath string
+}
+
+func (c *clangMatchUnderPath) Match(astNodeFilename string) bool {
+	if astNodeFilename == "" {
+		return true
+	}
+	return strings.HasPrefix(astNodeFilename, c.basePath)
+}
+
+//
+
+func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []string, matcher ClangMatcher) ([]interface{}, error) {
 
 	clangArgs := []string{`-x`, `c++`}
 	clangArgs = append(clangArgs, cflags...)
@@ -44,7 +64,7 @@ func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []strin
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		inner, innerErr = clangStripUpToFile(pr, inputHeader)
+		inner, innerErr = clangStripUpToFile(pr, matcher)
 	}()
 
 	err = cmd.Wait()
@@ -57,10 +77,10 @@ func clangExec(ctx context.Context, clangBin, inputHeader string, cflags []strin
 	return inner, innerErr
 }
 
-func mustClangExec(ctx context.Context, clangBin, inputHeader string, cflags []string) []interface{} {
+func mustClangExec(ctx context.Context, clangBin, inputHeader string, cflags []string, matcher ClangMatcher) []interface{} {
 
 	for i := 0; i < ClangMaxRetries; i++ {
-		astInner, err := clangExec(ctx, clangBin, inputHeader, cflags)
+		astInner, err := clangExec(ctx, clangBin, inputHeader, cflags, matcher)
 		if err != nil {
 			// Log and continue with next retry
 			log.Printf("WARNING: Clang execution failed: %v", err)
@@ -83,7 +103,7 @@ func mustClangExec(ctx context.Context, clangBin, inputHeader string, cflags []s
 // This cleans out everything in the translation unit that came from an
 // #included file.
 // @ref https://stackoverflow.com/a/71128654
-func clangStripUpToFile(stdout io.Reader, inputFilePath string) ([]interface{}, error) {
+func clangStripUpToFile(stdout io.Reader, matcher ClangMatcher) ([]interface{}, error) {
 
 	var obj = map[string]interface{}{}
 	err := json.NewDecoder(stdout).Decode(&obj)
@@ -108,10 +128,8 @@ func clangStripUpToFile(stdout io.Reader, inputFilePath string) ([]interface{},
 			return nil, errors.New("entry is not a map")
 		}
 
-		if _, ok := entry["isImplicit"]; ok {
-			// Don't keep
-			continue
-		}
+		// Check where this AST node came from, if it was directly written
+		// in this header or if it as part of an #include
 
 		var match_filename = ""
 
@@ -140,16 +158,13 @@ func clangStripUpToFile(stdout io.Reader, inputFilePath string) ([]interface{},
 
 		// log.Printf("# name=%v kind=%v filename=%q\n", entry["name"], entry["kind"], match_filename)
 
-		if match_filename == "" {
+		if matcher(match_filename) {
 			// Keep
 			ret = append(ret, entry)
-
-		} else if match_filename != inputFilePath {
-			// Skip this
-		} else {
-			// Keep this
-			// ret = append(ret, entry)
 		}
+
+		// Otherwise, discard this AST node, it comes from some imported file
+		// that we will likely scan separately
 	}
 
 	return ret, nil

cmd/genbindings/exceptions.go

@@ -24,6 +24,7 @@ func InsertTypedefs() {
 	// QFile doesn't see QFileDevice parent class enum
 	KnownTypedefs["QFile::Permissions"] = lookupResultTypedef{"qt", CppTypedef{"QFile::Permissions", parseSingleTypeString("QFileDevice::Permissions")}}
 	KnownTypedefs["QFileDevice::Permissions"] = lookupResultTypedef{"qt", CppTypedef{"QFile::Permissions", parseSingleTypeString("QFlags<QFileDevice::Permission>")}}
+
 }
 
 func AllowHeader(fullpath string) bool {
@@ -69,6 +70,11 @@ func ImportHeaderForClass(className string) bool {
 		return false
 	}
 
+	if strings.HasPrefix(className, "Qsci") {
+		// QScintilla - does not produce imports
+		return false
+	}
+
 	switch className {
 	case "QGraphicsEffectSource", // e.g. qgraphicseffect.h
 		"QAbstractConcatenable", // qstringbuilder.h
@@ -90,6 +96,10 @@ func AllowClass(className string) bool {
 		return false
 	}
 
+	if strings.HasPrefix(className, `std::`) {
+		return false // Scintilla bindings find some of these
+	}
+
 	switch className {
 	case
 		"QTextStreamManipulator", // Only seems to contain garbage methods
@@ -130,7 +140,6 @@ func AllowMethod(mm CppMethod) error {
 	}
 
 	return nil // OK, allow
-
 }
 
 func CheckComplexity(p CppParameter, isReturnType bool) error {
@@ -150,6 +159,12 @@ func CheckComplexity(p CppParameter, isReturnType bool) error {
 		if err := CheckComplexity(t, isReturnType); err != nil { // e.g. QGradientStops is a QVector<> (OK) of QGradientStop (not OK)
 			return err
 		}
+
+		// qsciscintilla.h QsciScintilla_Annotate4: no copy ctor for private type QsciStyledText
+		// Works fine normally, but not in a list
+		if t.ParameterType == "QsciStyledText" {
+			return ErrTooComplex
+		}
 	}
 
 	if strings.Contains(p.ParameterType, "(*)") { // Function pointer.
@@ -176,12 +191,16 @@ func CheckComplexity(p CppParameter, isReturnType bool) error {
 	if strings.HasPrefix(p.ParameterType, "QUrlTwoFlags<") {
 		return ErrTooComplex // e.g. qurl.h
 	}
+	if strings.HasPrefix(p.ParameterType, "FillResult<") {
+		return ErrTooComplex // Scintilla
+	}
 	if strings.HasPrefix(p.ParameterType, "std::") {
 		// std::initializer           e.g. qcborarray.h
 		// std::string                QByteArray->toStdString(). There are QString overloads already
 		// std::nullptr_t             Qcborstreamwriter
 		// std::chrono::nanoseconds   QDeadlineTimer_RemainingTimeAsDuration
 		// std::seed_seq              QRandom
+		// std::exception             Scintilla
 		return ErrTooComplex
 	}
 	if strings.Contains(p.ParameterType, `Iterator::value_type`) {
@@ -253,6 +272,7 @@ func CheckComplexity(p CppParameter, isReturnType bool) error {
 		"QXmlStreamNamespaceDeclarations", // e.g. qxmlstream.h. As above
 		"QXmlStreamNotationDeclarations",  // e.g. qxmlstream.h. As above
 		"QXmlStreamAttributes",            // e.g. qxmlstream.h
+		"LineLayout::ValidLevel",          // ..
 		"QtMsgType",                       // e.g. qdebug.h TODO Defined in qlogging.h, but omitted because it's predefined in qglobal.h, and our clangexec is too agressive
 		"QTextStreamFunction",             // e.g. qdebug.h
 		"QFactoryInterface",               // qfactoryinterface.h

cmd/genbindings/intermediate.go

@@ -110,6 +110,10 @@ func (p CppParameter) QtClassType() bool {
 		return true
 	}
 
+	if p.ParameterType == "Scintilla::Internal::Point" {
+		return true
+	}
+
 	if p.ParameterType == "QString" || p.ParameterType == "QByteArray" {
 		return true
 	}

cmd/genbindings/main.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -22,6 +23,10 @@ func importPathForQtPackage(packageName string) string {
 	switch packageName {
 	case "qt":
 		return BaseModule + "/qt"
+	case "qscintilla":
+		return BaseModule + "/qt-restricted-extras/" + packageName
+	case "scintillaedit":
+		return BaseModule + "/qt-extras/" + packageName
 	default:
 		return BaseModule + "/qt/" + packageName
 	}
@@ -87,9 +92,19 @@ func cleanGeneratedFilesInDir(dirpath string) {
 	log.Printf("Removed %d file(s).", cleaned)
 }
 
+func pkgConfigCflags(packageName string) string {
+	stdout, err := exec.Command(`pkg-config`, `--cflags`, packageName).Output()
+	if err != nil {
+		panic(err)
+	}
+
+	return string(stdout)
+}
+
 func main() {
 	clang := flag.String("clang", "clang", "Custom path to clang")
 	outDir := flag.String("outdir", "../../", "Output directory for generated gen_** files")
+	extraLibsDir := flag.String("extralibs", "/usr/local/src/", "Base directory to find extra library checkouts")
 
 	flag.Parse()
 
@@ -101,9 +116,9 @@ func main() {
 			"/usr/include/x86_64-linux-gnu/qt5/QtWidgets",
 		},
 		*clang,
-		// pkg-config --cflags Qt5Widgets
-		strings.Fields(`-DQT_WIDGETS_LIB -I/usr/include/x86_64-linux-gnu/qt5/QtWidgets -I/usr/include/x86_64-linux-gnu/qt5 -I/usr/include/x86_64-linux-gnu/qt5/QtCore -DQT_GUI_LIB -I/usr/include/x86_64-linux-gnu/qt5/QtGui -DQT_CORE_LIB`),
+		strings.Fields(pkgConfigCflags("Qt5Widgets")),
 		filepath.Join(*outDir, "qt"),
+		ClangMatchSameHeaderDefinitionOnly,
 	)
 
 	generate(
@@ -112,17 +127,45 @@ func main() {
 			"/usr/include/x86_64-linux-gnu/qt5/QtPrintSupport",
 		},
 		*clang,
-		// pkg-config --cflags Qt5PrintSupport
-		strings.Fields(`-DQT_PRINTSUPPORT_LIB -I/usr/include/x86_64-linux-gnu/qt5/QtPrintSupport -I/usr/include/x86_64-linux-gnu/qt5 -I/usr/include/x86_64-linux-gnu/qt5/QtCore -I/usr/include/x86_64-linux-gnu/qt5/QtGui -DQT_WIDGETS_LIB -I/usr/include/x86_64-linux-gnu/qt5/QtWidgets -DQT_GUI_LIB -DQT_CORE_LIB`),
+		strings.Fields(pkgConfigCflags("Qt5PrintSupport")),
 		filepath.Join(*outDir, "qt/qprintsupport"),
+		ClangMatchSameHeaderDefinitionOnly,
+	)
+
+	// Depends on QtCore/Gui/Widgets, QPrintSupport
+	generate(
+		"qscintilla",
+		[]string{
+			"/usr/include/x86_64-linux-gnu/qt5/Qsci",
+		},
+		*clang,
+		strings.Fields(pkgConfigCflags("Qt5PrintSupport")),
+		filepath.Join(*outDir, "qt-restricted-extras/qscintilla"),
+		ClangMatchSameHeaderDefinitionOnly,
+	)
+
+	// Depends on QtCore/Gui/Widgets
+	generate(
+		"scintillaedit",
+		[]string{
+			filepath.Join(*extraLibsDir, "scintilla/qt/ScintillaEdit/ScintillaEdit.h"),
+		},
+		*clang,
+		strings.Fields("--std=c++1z "+pkgConfigCflags("ScintillaEdit")),
+		filepath.Join(*outDir, "qt-extras/scintillaedit"),
+		(&clangMatchUnderPath{filepath.Join(*extraLibsDir, "scintilla")}).Match,
 	)
 }
 
-func generate(packageName string, srcDirs []string, clangBin string, cflags []string, outDir string) {
+func generate(packageName string, srcDirs []string, clangBin string, cflags []string, outDir string, matcher ClangMatcher) {
 
 	var includeFiles []string
 	for _, srcDir := range srcDirs {
-		includeFiles = append(includeFiles, findHeadersInDir(srcDir)...)
+		if strings.HasSuffix(srcDir, `.h`) {
+			includeFiles = append(includeFiles, srcDir) // single .h
+		} else {
+			includeFiles = append(includeFiles, findHeadersInDir(srcDir)...)
+		}
 	}
 
 	log.Printf("Found %d header files to process.", len(includeFiles))
@@ -140,7 +183,7 @@ func generate(packageName string, srcDirs []string, clangBin string, cflags []st
 	// PASS 0 (Fill clang cache)
 	//
 
-	generateClangCaches(includeFiles, clangBin, cflags)
+	generateClangCaches(includeFiles, clangBin, cflags, matcher)
 
 	// The cache should now be fully populated.
 
@@ -267,7 +310,7 @@ func generate(packageName string, srcDirs []string, clangBin string, cflags []st
 	log.Printf("Processing %d file(s) completed", len(includeFiles))
 }
 
-func generateClangCaches(includeFiles []string, clangBin string, cflags []string) {
+func generateClangCaches(includeFiles []string, clangBin string, cflags []string, matcher ClangMatcher) {
 
 	var clangChan = make(chan string, 0)
 	var clangWg sync.WaitGroup
@@ -289,7 +332,7 @@ func generateClangCaches(includeFiles []string, clangBin string, cflags []string
 
 				// Parse the file
 				// This seems to intermittently fail, so allow retrying
-				astInner := mustClangExec(ctx, clangBin, inputHeader, cflags)
+				astInner := mustClangExec(ctx, clangBin, inputHeader, cflags, matcher)
 
 				// Write to cache
 				jb, err := json.MarshalIndent(astInner, "", "\t")