Re-write string.format for more feature/performance

Since this function is the core driver of template strings, it stands to reason that it should be as efficient as possible, and be capable of using as many fast-pass runtime features as can be.

Additionally, this adds a whole expression stack to format string arguments, pretty much enabling the use of arbitrarily complex expressions within template string arguments.

.vs/deemon-v141.sln

@@ -382,6 +382,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{2360741E-F
 		..\util\test\deemon-int-tostr.dee = ..\util\test\deemon-int-tostr.dee
 		..\util\test\deemon-java-lambda.dee = ..\util\test\deemon-java-lambda.dee
 		..\util\test\deemon-kwcall-reference-loophole.dee = ..\util\test\deemon-kwcall-reference-loophole.dee
+		..\util\test\deemon-list-setrange.dee = ..\util\test\deemon-list-setrange.dee
 		..\util\test\deemon-list-sort.dee = ..\util\test\deemon-list-sort.dee
 		..\util\test\deemon-mapping-byattr.dee = ..\util\test\deemon-mapping-byattr.dee
 		..\util\test\deemon-mapping-byhash.dee = ..\util\test\deemon-mapping-byhash.dee

.vs/deemon-v141.vcxproj

@@ -281,6 +281,8 @@
 		<ClInclude Include="..\src\deemon\objects\unicode\bytes_split.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\codec.h" />
 		<ClInclude Include="..\src\deemon\objects\unicode\finder.c.inl" />
+		<ClInclude Include="..\src\deemon\objects\unicode\format-expr.c.inl" />
+		<ClInclude Include="..\src\deemon\objects\unicode\format-impl.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\ordinals.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\regroups.h" />
 		<ClInclude Include="..\src\deemon\objects\unicode\reproxy.c.inl" />

.vs/deemon-v141.vcxproj.filters

@@ -816,6 +816,12 @@
 		<ClInclude Include="..\src\deemon\objects\unicode\finder.c.inl">
 			<Filter>src\objects\unicode</Filter>
 		</ClInclude>
+		<ClInclude Include="..\src\deemon\objects\unicode\format-expr.c.inl">
+			<Filter>src\objects\unicode</Filter>
+		</ClInclude>
+		<ClInclude Include="..\src\deemon\objects\unicode\format-impl.c.inl">
+			<Filter>src\objects\unicode</Filter>
+		</ClInclude>
 		<ClInclude Include="..\src\deemon\objects\unicode\ordinals.c.inl">
 			<Filter>src\objects\unicode</Filter>
 		</ClInclude>

.vs/deemon-v142.sln

@@ -382,6 +382,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "test", "test", "{2360741E-F
 		..\util\test\deemon-int-tostr.dee = ..\util\test\deemon-int-tostr.dee
 		..\util\test\deemon-java-lambda.dee = ..\util\test\deemon-java-lambda.dee
 		..\util\test\deemon-kwcall-reference-loophole.dee = ..\util\test\deemon-kwcall-reference-loophole.dee
+		..\util\test\deemon-list-setrange.dee = ..\util\test\deemon-list-setrange.dee
 		..\util\test\deemon-list-sort.dee = ..\util\test\deemon-list-sort.dee
 		..\util\test\deemon-mapping-byattr.dee = ..\util\test\deemon-mapping-byattr.dee
 		..\util\test\deemon-mapping-byhash.dee = ..\util\test\deemon-mapping-byhash.dee

.vs/deemon-v142.vcxproj

@@ -281,6 +281,8 @@
 		<ClInclude Include="..\src\deemon\objects\unicode\bytes_split.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\codec.h" />
 		<ClInclude Include="..\src\deemon\objects\unicode\finder.c.inl" />
+		<ClInclude Include="..\src\deemon\objects\unicode\format-expr.c.inl" />
+		<ClInclude Include="..\src\deemon\objects\unicode\format-impl.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\ordinals.c.inl" />
 		<ClInclude Include="..\src\deemon\objects\unicode\regroups.h" />
 		<ClInclude Include="..\src\deemon\objects\unicode\reproxy.c.inl" />

.vs/deemon-v142.vcxproj.filters

@@ -816,6 +816,12 @@
 		<ClInclude Include="..\src\deemon\objects\unicode\finder.c.inl">
 			<Filter>src\objects\unicode</Filter>
 		</ClInclude>
+		<ClInclude Include="..\src\deemon\objects\unicode\format-expr.c.inl">
+			<Filter>src\objects\unicode</Filter>
+		</ClInclude>
+		<ClInclude Include="..\src\deemon\objects\unicode\format-impl.c.inl">
+			<Filter>src\objects\unicode</Filter>
+		</ClInclude>
 		<ClInclude Include="..\src\deemon\objects\unicode\ordinals.c.inl">
 			<Filter>src\objects\unicode</Filter>
 		</ClInclude>

include/deemon/api.h

@@ -406,6 +406,15 @@ __pragma_GCC_diagnostic_ignored(Wstringop_overread)
 #endif
 #endif /* !CONFIG_[NO_]EXPERIMENTAL_NEW_MAPPING_OPERATORS */
 
+#if (!defined(CONFIG_EXPERIMENTAL_NEW_STRING_FORMAT) && \
+     !defined(CONFIG_NO_EXPERIMENTAL_NEW_STRING_FORMAT))
+#if 1 /* Not ready, yet */
+#define CONFIG_EXPERIMENTAL_NEW_STRING_FORMAT
+#else
+#define CONFIG_NO_EXPERIMENTAL_NEW_STRING_FORMAT
+#endif
+#endif /* !CONFIG_[NO_]EXPERIMENTAL_NEW_STRING_FORMAT */
+
 
 
 #ifdef CONFIG_HOST_WINDOWS

include/deemon/int.h

@@ -760,6 +760,11 @@ DFUNDEF WUNUSED NONNULL((1, 4)) int (DCALL Dee_Atou64)(/*utf-8*/ char const *__r
 	 sizeof(T) <= 2 ? Dee_Atoi16(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int16_t *)(value)) : \
 	 sizeof(T) <= 4 ? Dee_Atoi32(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int32_t *)(value)) : \
 	                  Dee_Atoi64(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int64_t *)(value)))
+#define Dee_TAtoiu(str, len, radix_and_flags, value)                                \
+	(sizeof(T) <= 1 ? Dee_Atoi8(str, len, (radix_and_flags), (int8_t *)(value)) :   \
+	 sizeof(T) <= 2 ? Dee_Atoi16(str, len, (radix_and_flags), (int16_t *)(value)) : \
+	 sizeof(T) <= 4 ? Dee_Atoi32(str, len, (radix_and_flags), (int32_t *)(value)) : \
+	                  Dee_Atoi64(str, len, (radix_and_flags), (int64_t *)(value)))
 #else /* __NO_builtin_choose_expr */
 #define DEE_PRIVATE_ATOI_FLAGS(T, flags) \
 	__builtin_choose_expr(((T)-1) < (T)0, (flags) | DEEATOI_STRING_FSIGNED, (flags))
@@ -768,7 +773,14 @@ DFUNDEF WUNUSED NONNULL((1, 4)) int (DCALL Dee_Atou64)(/*utf-8*/ char const *__r
 	__builtin_choose_expr(sizeof(T) <= 2, Dee_Atoi16(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int16_t *)(value)),  \
 	__builtin_choose_expr(sizeof(T) <= 4, Dee_Atoi32(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int32_t *)(value)),  \
 	                                      Dee_Atoi64(str, len, DEE_PRIVATE_ATOI_FLAGS(T, radix_and_flags), (int64_t *)(value)))))
+#define Dee_TAtoiu(str, len, radix_and_flags, value)                                                           \
+	__builtin_choose_expr(sizeof(*(value)) <= 1, Dee_Atoi8(str, len, (radix_and_flags), (int8_t *)(value)),    \
+	__builtin_choose_expr(sizeof(*(value)) <= 2, Dee_Atoi16(str, len, (radix_and_flags), (int16_t *)(value)),  \
+	__builtin_choose_expr(sizeof(*(value)) <= 4, Dee_Atoi32(str, len, (radix_and_flags), (int32_t *)(value)),  \
+	                                             Dee_Atoi64(str, len, (radix_and_flags), (int64_t *)(value)))))
 #endif /* !__NO_builtin_choose_expr */
+#define Dee_TAtois(str, len, radix_and_flags, value)                                                                                    \
+	Dee_TAtoiu(str, len, (radix_and_flags) | DEEATOI_STRING_FSIGNED, value)
 
 
 /* Print an integer to a given format-printer.

include/deemon/object.h

@@ -2008,6 +2008,48 @@ typedef WUNUSED_T NONNULL_T((2, 3)) Dee_ssize_t (DCALL *Dee_foreach_pair_t)(void
 typedef WUNUSED_T NONNULL_T((2)) Dee_ssize_t (DCALL *Dee_enumerate_t)(void *arg, DeeObject *index, /*nullable*/ DeeObject *value);
 typedef WUNUSED_T Dee_ssize_t (DCALL *Dee_enumerate_index_t)(void *arg, size_t index, /*nullable*/ DeeObject *value);
 
+#if 0
+struct my_foreach_data {
+};
+
+PRIVATE WUNUSED NONNULL((2)) Dee_ssize_t DCALL
+my_foreach_cb(void *arg, DeeObject *elem) {
+	struct my_foreach_data *data;
+	data = (struct my_foreach_data *)arg;
+	return DeeError_NOTIMPLEMENTED();
+}
+
+struct my_foreach_pair_data {
+};
+
+PRIVATE WUNUSED NONNULL((2, 3)) Dee_ssize_t DCALL
+my_foreach_pair_cb(void *arg, DeeObject *key, DeeObject *value) {
+	struct my_foreach_pair_data *data;
+	data = (struct my_foreach_pair_data *)arg;
+	return DeeError_NOTIMPLEMENTED();
+}
+
+struct my_enumerate_data {
+};
+
+PRIVATE WUNUSED NONNULL((2)) Dee_ssize_t DCALL
+my_enumerate_cb(void *arg, DeeObject *index, /*nullable*/ DeeObject *value) {
+	struct my_enumerate_data *data;
+	data = (struct my_enumerate_data *)arg;
+	return DeeError_NOTIMPLEMENTED();
+}
+
+struct my_enumerate_index_data {
+};
+
+PRIVATE WUNUSED Dee_ssize_t DCALL
+my_enumerate_index_cb(void *arg, size_t index, /*nullable*/ DeeObject *value) {
+	struct my_enumerate_index_data *data;
+	data = (struct my_enumerate_index_data *)arg;
+	return DeeError_NOTIMPLEMENTED();
+}
+#endif
+
 struct Dee_type_nsi;
 struct Dee_type_seq_cache;
 struct Dee_type_seq {

include/deemon/string.h

@@ -2153,6 +2153,26 @@ DeeString_NewWithWidth(void const *__restrict str,
 
 
 
+/* Generic format printer function (can be used to implement
+ * both string (unicode), as well as Bytes formatting)
+ * @param: pattern_printer: Printer used to emit data from `pattern' (i.e. static data)
+ *                          Note that this is a distinct argument so-as to allow `pattern'
+ *                          to be an ASCII string, rather than it needing to be utf-8.
+ * @param: data_printer:    Printer used to emit data from `args' (i.e. dynamic data) */
+DFUNDEF WUNUSED NONNULL((1, 3)) Dee_ssize_t DCALL
+DeeString_FormatPrinter(char const *pattern, size_t pattern_length, DeeObject *args,
+                        Dee_formatprinter_t pattern_printer,
+                        Dee_formatprinter_t data_printer, void *arg);
+
+/* API functions for `string.format'. */
+DFUNDEF WUNUSED NONNULL((1, 2)) DREF DeeObject *DFCALL
+DeeString_FormatWStr(/*utf-8*/ char const *pattern_wstr, DeeObject *args);
+DFUNDEF WUNUSED NONNULL((1, 2)) DREF DeeObject *DFCALL
+DeeString_Format(DeeObject *pattern, DeeObject *args);
+
+
+
+
 /* Return a string containing a single character. */
 #if defined(__INTELLISENSE__) && defined(__cplusplus)
 extern "C++" {

src/deemon/linker-scripts/link-deemon-gcc-i386-cygwin.def

@@ -1052,6 +1052,7 @@ EXPORTS
 	_DeeString_Chr32@4=DeeString_Chr32@4
 	_DeeString_Chr8@4=DeeString_Chr8@4
 	_DeeString_DecodeBackslashEscaped@16=DeeString_DecodeBackslashEscaped@16
+	_DeeString_FormatPrinter@24=DeeString_FormatPrinter@24
 	_DeeString_FreeWidth@4=DeeString_FreeWidth@4
 	_DeeString_FromBackslashEscaped@12=DeeString_FromBackslashEscaped@12
 	_DeeString_GetChar@8=DeeString_GetChar@8

src/deemon/linker-scripts/link-deemon-msvc-i386-win32.def

@@ -1052,6 +1052,7 @@ EXPORTS
 	DeeString_Chr32@4=_DeeString_Chr32@4
 	DeeString_Chr8@4=_DeeString_Chr8@4
 	DeeString_DecodeBackslashEscaped@16=_DeeString_DecodeBackslashEscaped@16
+	DeeString_FormatPrinter@24=_DeeString_FormatPrinter@24
 	DeeString_FreeWidth@4=_DeeString_FreeWidth@4
 	DeeString_FromBackslashEscaped@12=_DeeString_FromBackslashEscaped@12
 	DeeString_GetChar@8=_DeeString_GetChar@8

src/deemon/objects/none.c

@@ -360,10 +360,10 @@ PRIVATE struct type_seq none_seq = {
 	/* .tp_delrange                     = */ (int (DCALL *)(DeeObject *, DeeObject *, DeeObject *))&none_i3,
 	/* .tp_setrange                     = */ (int (DCALL *)(DeeObject *, DeeObject *, DeeObject *, DeeObject *))&none_i4,
 	/* .tp_nsi                          = */ &none_nsi,
-	/* .tp_foreach                      = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_foreach_t, void *))&none_s3,
-	/* .tp_foreach_pair                 = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_foreach_pair_t, void *))&none_s3,
-	/* .tp_enumerate                    = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_enumerate_t, void *))&none_s3,
-	/* .tp_enumerate_index              = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_enumerate_index_t, void *, size_t, size_t))&none_s5,
+	/* .tp_foreach                      = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_foreach_t, void *))&none_s3, /* TODO: Infinite loop */
+	/* .tp_foreach_pair                 = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_foreach_pair_t, void *))&none_s3, /* TODO: Infinite loop */
+	/* .tp_enumerate                    = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_enumerate_t, void *))&none_s3, /* TODO: Infinite loop */
+	/* .tp_enumerate_index              = */ (Dee_ssize_t (DCALL *)(DeeObject *__restrict, Dee_enumerate_index_t, void *, size_t, size_t))&none_s5, /* TODO: Infinite loop */
 	/* .tp_iterkeys                     = */ (DREF DeeObject *(DCALL *)(DeeObject *__restrict))&none_1,
 	/* .tp_bounditem                    = */ (int (DCALL *)(DeeObject *, DeeObject *))&none_i2_1,
 	/* .tp_hasitem                      = */ (int (DCALL *)(DeeObject *, DeeObject *))&none_i2_1,

src/deemon/objects/unicode/bytes_functions.c.inl

@@ -778,11 +778,31 @@ err:
 	return NULL;
 }
 
+
+#ifdef CONFIG_EXPERIMENTAL_NEW_STRING_FORMAT
+PRIVATE WUNUSED NONNULL((1)) DREF DeeObject *DCALL
+bytes_format(Bytes *self, size_t argc, DeeObject *const *argv) {
+	DeeObject *args;
+	struct bytes_printer printer;
+	if (DeeArg_Unpack(argc, argv, "o:format", &args))
+		goto err;
+	bytes_printer_init(&printer);
+	if unlikely(DeeString_FormatPrinter((char const *)DeeBytes_DATA(self), DeeBytes_SIZE(self), args,
+	                                    (dformatprinter)&bytes_printer_append, &bytes_printer_print,
+	                                    &printer) < 0)
+			goto err_printer;
+	return bytes_printer_pack(&printer);
+err_printer:
+	bytes_printer_fini(&printer);
+err:
+	return NULL;
+}
+#else /* CONFIG_EXPERIMENTAL_NEW_STRING_FORMAT */
 INTDEF dssize_t DCALL
-DeeBytes_Format(dformatprinter printer,
-                dformatprinter format_printer, void *arg,
-                char const *__restrict format,
-                size_t format_len, DeeObject *__restrict args);
+DeeBytes_Format_old(dformatprinter printer,
+                    dformatprinter format_printer, void *arg,
+                    char const *__restrict format,
+                    size_t format_len, DeeObject *__restrict args);
 
 PRIVATE WUNUSED NONNULL((1)) DREF DeeObject *DCALL
 bytes_format(Bytes *self, size_t argc, DeeObject *const *argv) {
@@ -791,7 +811,7 @@ bytes_format(Bytes *self, size_t argc, DeeObject *const *argv) {
 		goto err;
 	{
 		struct bytes_printer printer = BYTES_PRINTER_INIT;
-		if unlikely(DeeBytes_Format(&bytes_printer_print,
+		if unlikely(DeeBytes_Format_old(&bytes_printer_print,
 		                            (dformatprinter)&bytes_printer_append,
 		                            &printer,
 		                            (char *)DeeBytes_DATA(self),
@@ -805,6 +825,7 @@ err_printer:
 err:
 	return NULL;
 }
+#endif /* !CONFIG_EXPERIMENTAL_NEW_STRING_FORMAT */