From a7fb6eed93e273bf8e75560c6477a6f6f3b520c9 Mon Sep 17 00:00:00 2001 From: Daco Harkes Date: Fri, 10 Jan 2025 16:00:23 +0100 Subject: [PATCH] [native_assets_cli] Fix system libraries and add documentation (#1880) Closes: https://github.com/dart-lang/native/issues/940 Closes: https://github.com/dart-lang/native/issues/1879 --- .github/workflows/native.yaml | 12 ++++- .../build_runner/system_library_test.dart | 37 ++++++++++++++ .../test_data/manifest.yaml | 6 +++ .../test_data/system_library/hook/build.dart | 47 ++++++++++++++++++ .../system_library/lib/memory_executable.dart | 17 +++++++ .../system_library/lib/memory_process.dart | 17 +++++++ .../system_library/lib/memory_system.dart | 17 +++++++ .../test_data/system_library/pubspec.yaml | 22 +++++++++ .../system_library/test/memory_test.dart | 49 +++++++++++++++++++ .../native_assets_cli/example/build/README.md | 3 ++ .../example/build/system_library/README.md | 21 ++++++++ .../build/system_library/hook/build.dart | 29 +++++++++++ .../build/system_library/lib/memory.dart | 21 ++++++++ .../example/build/system_library/pubspec.yaml | 22 +++++++++ .../system_library/test/memory_test.dart | 23 +++++++++ .../lib/src/code_assets/validation.dart | 11 ++++- 16 files changed, 352 insertions(+), 2 deletions(-) create mode 100644 pkgs/native_assets_builder/test/build_runner/system_library_test.dart create mode 100644 pkgs/native_assets_builder/test_data/system_library/hook/build.dart create mode 100644 pkgs/native_assets_builder/test_data/system_library/lib/memory_executable.dart create mode 100644 pkgs/native_assets_builder/test_data/system_library/lib/memory_process.dart create mode 100644 pkgs/native_assets_builder/test_data/system_library/lib/memory_system.dart create mode 100644 pkgs/native_assets_builder/test_data/system_library/pubspec.yaml create mode 100644 pkgs/native_assets_builder/test_data/system_library/test/memory_test.dart create mode 100644 pkgs/native_assets_cli/example/build/system_library/README.md create mode 100644 pkgs/native_assets_cli/example/build/system_library/hook/build.dart create mode 100644 pkgs/native_assets_cli/example/build/system_library/lib/memory.dart create mode 100644 pkgs/native_assets_cli/example/build/system_library/pubspec.yaml create mode 100644 pkgs/native_assets_cli/example/build/system_library/test/memory_test.dart diff --git a/.github/workflows/native.yaml b/.github/workflows/native.yaml index 123e92aa1..a4570e2b7 100644 --- a/.github/workflows/native.yaml +++ b/.github/workflows/native.yaml @@ -120,12 +120,14 @@ jobs: - run: dart pub get -C example/build/use_dart_api/ if: ${{ matrix.package == 'native_assets_cli' }} + - run: dart pub get -C example/build/system_library/ + if: ${{ matrix.package == 'native_assets_cli' }} + - run: dart pub get -C example/link/package_with_assets/ if: ${{ matrix.package == 'native_assets_cli' }} - run: dart pub get -C example/link/app_with_asset_treeshaking/ if: ${{ matrix.package == 'native_assets_cli' }} - - run: dart pub get -C test_data/fail_build/ if: ${{ matrix.package == 'native_assets_builder' }} @@ -135,6 +137,9 @@ jobs: - run: dart pub get -C test_data/depend_on_fail_build_app/ if: ${{ matrix.package == 'native_assets_builder' }} + - run: dart pub get -C test_data/system_library/ + if: ${{ matrix.package == 'native_assets_builder' }} + - run: dart analyze --fatal-infos # Run on dev to ensure we're not depending on deprecated SDK things. @@ -172,6 +177,11 @@ jobs: - run: dart --enable-experiment=native-assets test working-directory: pkgs/${{ matrix.package }}/example/build/use_dart_api/ if: ${{ matrix.package == 'native_assets_cli' && matrix.sdk == 'dev' && !matrix.breaking-change }} + + # TODO(https://github.com/dart-lang/native/issues/1879): Enable this when the fix has rolled into Dart. + # - run: dart --enable-experiment=native-assets test + # working-directory: pkgs/${{ matrix.package }}/example/build/system_library/ + # if: ${{ matrix.package == 'native_assets_cli' && matrix.sdk == 'dev' && !matrix.breaking-change }} - name: Install coverage run: dart pub global activate coverage diff --git a/pkgs/native_assets_builder/test/build_runner/system_library_test.dart b/pkgs/native_assets_builder/test/build_runner/system_library_test.dart new file mode 100644 index 000000000..e962b7b94 --- /dev/null +++ b/pkgs/native_assets_builder/test/build_runner/system_library_test.dart @@ -0,0 +1,37 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:test/test.dart'; + +import '../helpers.dart'; +import 'helpers.dart'; + +const Timeout longTimeout = Timeout(Duration(minutes: 5)); + +void main() async { + test('system library', timeout: longTimeout, () async { + await inTempDir((tempUri) async { + await copyTestProjects(targetUri: tempUri); + final packageUri = tempUri.resolve('system_library/'); + + await runPubGet( + workingDirectory: packageUri, + logger: logger, + ); + + final logMessages = []; + final result = (await build( + packageUri, + logger, + dartExecutable, + capturedLogs: logMessages, + inputValidator: validateCodeAssetBuildInput, + buildAssetTypes: [CodeAsset.type], + buildValidator: validateCodeAssetBuildOutput, + applicationAssetValidator: validateCodeAssetInApplication, + ))!; + expect(result.encodedAssets.length, 3); + }); + }); +} diff --git a/pkgs/native_assets_builder/test_data/manifest.yaml b/pkgs/native_assets_builder/test_data/manifest.yaml index 43c8fcbcb..a429dab2b 100644 --- a/pkgs/native_assets_builder/test_data/manifest.yaml +++ b/pkgs/native_assets_builder/test_data/manifest.yaml @@ -157,3 +157,9 @@ - wrong_linker/pubspec.yaml - wrong_namespace_asset/hook/build.dart - wrong_namespace_asset/pubspec.yaml +- system_library/pubspec.yaml +- system_library/hook/build.dart +- system_library/lib/memory_executable.dart +- system_library/lib/memory_process.dart +- system_library/lib/memory_system.dart +- system_library/test/memory_test.dart diff --git a/pkgs/native_assets_builder/test_data/system_library/hook/build.dart b/pkgs/native_assets_builder/test_data/system_library/hook/build.dart new file mode 100644 index 000000000..eeeb406fb --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/hook/build.dart @@ -0,0 +1,47 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:native_assets_cli/code_assets.dart'; + +void main(List arguments) async { + await build(arguments, (input, output) async { + final targetOS = input.config.code.targetOS; + final targetArchitecture = input.config.code.targetArchitecture; + output.assets.code.addAll([ + CodeAsset( + package: input.packageName, + name: 'memory_system.dart', + linkMode: DynamicLoadingSystem( + Uri.file( + switch (targetOS) { + OS.android => 'libc.so.6', + OS.iOS => 'libc.dylib', + OS.linux => 'libc.so.6', + OS.macOS => 'libc.dylib', + OS.windows => 'ole32.dll', + _ => + throw UnsupportedError('Unknown operating system: $targetOS'), + }, + ), + ), + os: targetOS, + architecture: targetArchitecture, + ), + CodeAsset( + package: input.packageName, + name: 'memory_executable.dart', + linkMode: LookupInExecutable(), + os: targetOS, + architecture: targetArchitecture, + ), + CodeAsset( + package: input.packageName, + name: 'memory_process.dart', + linkMode: LookupInProcess(), + os: targetOS, + architecture: targetArchitecture, + ), + ]); + }); +} diff --git a/pkgs/native_assets_builder/test_data/system_library/lib/memory_executable.dart b/pkgs/native_assets_builder/test_data/system_library/lib/memory_executable.dart new file mode 100644 index 000000000..73590535a --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/lib/memory_executable.dart @@ -0,0 +1,17 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; + +@Native() +external Pointer malloc(int size); + +@Native() +external void free(Pointer pointer); + +@Native() +external Pointer coTaskMemAlloc(int cb); + +@Native() +external void coTaskMemFree(Pointer pv); diff --git a/pkgs/native_assets_builder/test_data/system_library/lib/memory_process.dart b/pkgs/native_assets_builder/test_data/system_library/lib/memory_process.dart new file mode 100644 index 000000000..73590535a --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/lib/memory_process.dart @@ -0,0 +1,17 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; + +@Native() +external Pointer malloc(int size); + +@Native() +external void free(Pointer pointer); + +@Native() +external Pointer coTaskMemAlloc(int cb); + +@Native() +external void coTaskMemFree(Pointer pv); diff --git a/pkgs/native_assets_builder/test_data/system_library/lib/memory_system.dart b/pkgs/native_assets_builder/test_data/system_library/lib/memory_system.dart new file mode 100644 index 000000000..73590535a --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/lib/memory_system.dart @@ -0,0 +1,17 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; + +@Native() +external Pointer malloc(int size); + +@Native() +external void free(Pointer pointer); + +@Native() +external Pointer coTaskMemAlloc(int cb); + +@Native() +external void coTaskMemFree(Pointer pv); diff --git a/pkgs/native_assets_builder/test_data/system_library/pubspec.yaml b/pkgs/native_assets_builder/test_data/system_library/pubspec.yaml new file mode 100644 index 000000000..747b28b25 --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/pubspec.yaml @@ -0,0 +1,22 @@ +name: system_library +description: Uses some functions from system lirbaries. +version: 0.1.0 + +publish_to: none + +environment: + sdk: '>=3.3.0 <4.0.0' + +dependencies: + logging: ^1.1.1 + # native_assets_cli: ^0.10.0 + native_assets_cli: + path: ../../../native_assets_cli/ + # native_toolchain_c: ^0.7.0 + native_toolchain_c: + path: ../../../native_toolchain_c/ + +dev_dependencies: + ffigen: ^10.0.0 + lints: ^3.0.0 + test: ^1.23.1 diff --git a/pkgs/native_assets_builder/test_data/system_library/test/memory_test.dart b/pkgs/native_assets_builder/test_data/system_library/test/memory_test.dart new file mode 100644 index 000000000..3d20f386e --- /dev/null +++ b/pkgs/native_assets_builder/test_data/system_library/test/memory_test.dart @@ -0,0 +1,49 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:io'; + +import 'package:system_library/memory_executable.dart' as executable; +import 'package:system_library/memory_process.dart' as process; +import 'package:system_library/memory_system.dart' as system; +import 'package:test/test.dart'; + +void main() { + test('executable', () { + if (Platform.isWindows) { + final pointer = executable.coTaskMemAlloc(8); + expect(pointer, isNot(nullptr)); + executable.coTaskMemFree(pointer); + } else { + final pointer = executable.malloc(8); + expect(pointer, isNot(nullptr)); + executable.free(pointer); + } + }); + + test('process', () { + if (Platform.isWindows) { + final pointer = process.coTaskMemAlloc(8); + expect(pointer, isNot(nullptr)); + process.coTaskMemFree(pointer); + } else { + final pointer = process.malloc(8); + expect(pointer, isNot(nullptr)); + process.free(pointer); + } + }); + + test('system', () { + if (Platform.isWindows) { + final pointer = system.coTaskMemAlloc(8); + expect(pointer, isNot(nullptr)); + system.coTaskMemFree(pointer); + } else { + final pointer = system.malloc(8); + expect(pointer, isNot(nullptr)); + system.free(pointer); + } + }); +} diff --git a/pkgs/native_assets_cli/example/build/README.md b/pkgs/native_assets_cli/example/build/README.md index fad771d89..1b303f364 100644 --- a/pkgs/native_assets_cli/example/build/README.md +++ b/pkgs/native_assets_cli/example/build/README.md @@ -25,6 +25,9 @@ Examples: * [native_dynamic_linking/](native_dynamic_linking/) contains source code for 3 native libraries that depend on each other and load each other with the native dynamic loader at runtime. +* Using system libraries + * [system_library/](system_library/) contains a package using native libaries + available on the host system where a Dart or Flutter app is deployed. * Building dynamic libraries against `dart_api_dl.h`. * [use_dart_api/](use_dart_api/) contains a library with C code that invokes `dart_api_dl.h` to interact with the Dart runtime. diff --git a/pkgs/native_assets_cli/example/build/system_library/README.md b/pkgs/native_assets_cli/example/build/system_library/README.md new file mode 100644 index 000000000..048f91c78 --- /dev/null +++ b/pkgs/native_assets_cli/example/build/system_library/README.md @@ -0,0 +1,21 @@ +An example of a Dart library using a native system libary. + +## Note + +Note that most system libraries on operating systems will not be available as a +C API. On MacOS/iOS, FFIgen and Swiftgen will need to be used to access the APIs +only available in Objective-C or Swift. On Android, JNIgen will need to be used +to access the APIs only available in Java or Kotlin. This package only details +how to use C APIs. For using system APIs with FFIgen, JNIgen, and Swiftgen refer +to the documentation in these packages. + +## Usage + +Run tests with `dart --enable-experiment=native-assets test`. + +## Code organization + +A typical layout of a package which uses system libraries: + +* `hook/build.dart` declares the system libraries used. +* `lib/` contains Dart code which uses the system libraries. diff --git a/pkgs/native_assets_cli/example/build/system_library/hook/build.dart b/pkgs/native_assets_cli/example/build/system_library/hook/build.dart new file mode 100644 index 000000000..92753783a --- /dev/null +++ b/pkgs/native_assets_cli/example/build/system_library/hook/build.dart @@ -0,0 +1,29 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'package:native_assets_cli/code_assets.dart'; + +void main(List arguments) async { + await build(arguments, (input, output) async { + final targetOS = input.config.code.targetOS; + output.assets.code.add(CodeAsset( + package: input.packageName, + name: 'memory.dart', + linkMode: DynamicLoadingSystem( + Uri.file( + switch (targetOS) { + OS.android => 'libc.so.6', + OS.iOS => 'libc.dylib', + OS.linux => 'libc.so.6', + OS.macOS => 'libc.dylib', + OS.windows => 'ole32.dll', + _ => throw UnsupportedError('Unknown operating system: $targetOS'), + }, + ), + ), + os: targetOS, + architecture: input.config.code.targetArchitecture, + )); + }); +} diff --git a/pkgs/native_assets_cli/example/build/system_library/lib/memory.dart b/pkgs/native_assets_cli/example/build/system_library/lib/memory.dart new file mode 100644 index 000000000..5b15198f4 --- /dev/null +++ b/pkgs/native_assets_cli/example/build/system_library/lib/memory.dart @@ -0,0 +1,21 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +/// Example of using system libraries. +@DefaultAsset('package:system_library/memory.dart') +library; + +import 'dart:ffi'; + +@Native() +external Pointer malloc(int size); + +@Native() +external void free(Pointer pointer); + +@Native() +external Pointer coTaskMemAlloc(int cb); + +@Native() +external void coTaskMemFree(Pointer pv); diff --git a/pkgs/native_assets_cli/example/build/system_library/pubspec.yaml b/pkgs/native_assets_cli/example/build/system_library/pubspec.yaml new file mode 100644 index 000000000..52d8e7778 --- /dev/null +++ b/pkgs/native_assets_cli/example/build/system_library/pubspec.yaml @@ -0,0 +1,22 @@ +name: system_library +description: Uses some functions from system lirbaries. +version: 0.1.0 + +publish_to: none + +environment: + sdk: '>=3.3.0 <4.0.0' + +dependencies: + logging: ^1.1.1 + # native_assets_cli: ^0.10.0 + native_assets_cli: + path: ../../../../native_assets_cli/ + # native_toolchain_c: ^0.7.0 + native_toolchain_c: + path: ../../../../native_toolchain_c/ + +dev_dependencies: + ffigen: ^10.0.0 + lints: ^3.0.0 + test: ^1.23.1 diff --git a/pkgs/native_assets_cli/example/build/system_library/test/memory_test.dart b/pkgs/native_assets_cli/example/build/system_library/test/memory_test.dart new file mode 100644 index 000000000..988cf7a60 --- /dev/null +++ b/pkgs/native_assets_cli/example/build/system_library/test/memory_test.dart @@ -0,0 +1,23 @@ +// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:ffi'; +import 'dart:io'; + +import 'package:system_library/memory.dart'; +import 'package:test/test.dart'; + +void main() { + test('invoke native function', () { + if (Platform.isWindows) { + final pointer = coTaskMemAlloc(8); + expect(pointer, isNot(nullptr)); + coTaskMemFree(pointer); + } else { + final pointer = malloc(8); + expect(pointer, isNot(nullptr)); + free(pointer); + } + }); +} diff --git a/pkgs/native_assets_cli/lib/src/code_assets/validation.dart b/pkgs/native_assets_cli/lib/src/code_assets/validation.dart index 03825d0c9..7016e3799 100644 --- a/pkgs/native_assets_cli/lib/src/code_assets/validation.dart +++ b/pkgs/native_assets_cli/lib/src/code_assets/validation.dart @@ -198,7 +198,7 @@ void _validateCodeAssets( } final file = codeAsset.file; - if (file == null && !dryRun) { + if (file == null && !dryRun && _mustHaveFile(codeAsset.linkMode)) { errors.add('CodeAsset "$id" has no file.'); } if (file != null && !dryRun && !File.fromUri(file).existsSync()) { @@ -207,6 +207,15 @@ void _validateCodeAssets( } } +bool _mustHaveFile(LinkMode linkMode) => switch (linkMode) { + LookupInExecutable _ => false, + LookupInProcess _ => false, + DynamicLoadingSystem _ => false, + DynamicLoadingBundled _ => true, + StaticLinking _ => true, + _ => throw UnsupportedError('Unknown link mode: $linkMode.'), + }; + void _groupCodeAssetsByFilename( CodeAsset codeAsset, Map> fileNameToEncodedAssetId,